Information Retrieval - Image Event Detection - Paolo Cortis¶

Running experiments with different pipelines and machine learning models for a multi class image event classification task on soccer images.

Libraries¶

In [1]:
import os
import cv2
from skimage import transform
from tensorflow.python.ops.numpy_ops import np_config
np_config.enable_numpy_behavior()

import random
import pandas as pd
import numpy as np
from tqdm import tqdm

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
from tensorflow.keras import activations, layers, optimizers, losses
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras import Sequential
from extra_keras_metrics import get_standard_binary_metrics, get_minimal_multiclass_metrics
from sklearn.metrics import accuracy_score, average_precision_score, roc_auc_score
from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit

from sklearn.metrics import ConfusionMatrixDisplay
import seaborn as sns
from matplotlib import pyplot as plt
%matplotlib inline
In [2]:
soccer_images_dim = 80

Data Retrieval¶

Import of the main datasets used:

For model training and evaluation

  • Soccer Events: images of the soccer events to classify.
  • Generic Events: images of generic soccer scenes (not events) and random images.

For further model evaluation, more diverse soccer and random images

  • Real Soccer: few images of the soccer events (not from the football analysis dataset).
  • Test dataset: few soccer and random images (not from soccer and generic events dataset).
In [3]:
def loadData(root, rgb):
    """ Return images from the dataset with the corresponding label from a root directory.

    Parameters
    ------------------------
    root: str
        The root folder with dataset.
    rgb: bool
        Specify whether to import images in rgb or gray scale.
        
    Returns
    ------------------------
    data: dict
        A dictionary with data images.
    folder_labels: list
        A list with the images labels."""
    
    data = {}
    folder_labels = []
    
    for folder_name in os.listdir(root):
        folder_path = os.path.join(root, folder_name)
    
        # Check if the item is a directory
        if os.path.isdir(folder_path):
            folder_files = []
            
            for file_name in os.listdir(folder_path):
                img = cv2.imread(os.path.join(folder_path, file_name))
                if img is not None:
                    resizedImg = cv2.resize(img, (soccer_images_dim,soccer_images_dim), interpolation=cv2.INTER_CUBIC)
                    RGBImg = cv2.cvtColor(resizedImg, cv2.COLOR_BGR2RGB)
                    if rgb == False:
                        grayImg = cv2.cvtColor(RGBImg, cv2.COLOR_RGB2GRAY)
                        folder_files.append(grayImg)
                    else:
                        folder_files.append(RGBImg)
                    folder_labels.append(folder_name)

            data[folder_name] = folder_files

    return data, folder_labels
In [10]:
# Soccer events
folderSoccer = r'D:\MachineLearning\Datasets\Soccer\TrainTestSoccer'
# Generic soccer and random events
folderGenericEvents = r'D:\MachineLearning\Datasets\Soccer\GenericEvents'
# Broadly different soccer events images)
folderRSoccer = r'D:\MachineLearning\Datasets\Soccer\RealSoccer'
# RealSoccer + events and genericSoccer images
folderMyTestDataset = r'D:\MachineLearning\Datasets\Soccer\myDataset'

data_soccer, labels_soccer = loadData(folderSoccer, False)
data_soccer_rgb, _ = loadData(folderSoccer, True)
data_generic_events, labels_generic_events = loadData(folderGenericEvents, False)
real_data_soccer, real_labels_soccer = loadData(folderRSoccer, False)
data_myTest, labels_myTest = loadData(folderMyTestDataset, False)
In [14]:
# Save datasets
import pickle

# Save Dataset (save dictionary to data_soccer.pkl file)
with open('data/data_soccer.pkl', 'wb') as fp:
    pickle.dump(data_soccer, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to data_soccer_rgb.pkl file)
with open('data/data_soccer_rgb.pkl', 'wb') as fp:
    pickle.dump(data_soccer_rgb, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to labels_soccer.pkl file)
with open('data/labels_soccer.pkl', 'wb') as fp:
    pickle.dump(labels_soccer, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to data_generic_events.pkl file)
with open('data/data_generic_events.pkl', 'wb') as fp:
    pickle.dump(data_generic_events, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to labels_generic_events.pkl file)
with open('data/labels_generic_events.pkl', 'wb') as fp:
    pickle.dump(labels_generic_events, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to real_data_soccer.pkl file)
with open('data/real_data_soccer.pkl', 'wb') as fp:
    pickle.dump(real_data_soccer, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to real_labels_soccer.pkl file)
with open('data/real_labels_soccer.pkl', 'wb') as fp:
    pickle.dump(real_labels_soccer, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to data_myTest.pkl file)
with open('data/data_myTest.pkl', 'wb') as fp:
    pickle.dump(data_myTest, fp)
    print('dictionary saved successfully to file')
    
# Save Dataset (save dictionary to labels_myTest.pkl file)
with open('data/labels_myTest.pkl', 'wb') as fp:
    pickle.dump(labels_myTest, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
dictionary saved successfully to file
In [4]:
# Load datasets
import pickle

# Load Dataset (Read dictionary pkl file)
with open('data/data_soccer.pkl', 'rb') as fp:
    data_soccer = pickle.load(fp)
    
# Load Dataset (Read dictionary pkl file)
with open('data/data_soccer_rgb.pkl', 'rb') as fp:
    data_soccer_rgb = pickle.load(fp)
    
with open('data/labels_soccer.pkl', 'rb') as fp:
    labels_soccer = pickle.load(fp)
    
with open('data/data_generic_events.pkl', 'rb') as fp:
    data_generic_events = pickle.load(fp)
    
with open('data/labels_generic_events.pkl', 'rb') as fp:
    labels_generic_events = pickle.load(fp)
    
with open('data/real_data_soccer.pkl', 'rb') as fp:
    real_data_soccer = pickle.load(fp)
    
with open('data/real_labels_soccer.pkl', 'rb') as fp:
    real_labels_soccer = pickle.load(fp)
    
with open('data/data_myTest.pkl', 'rb') as fp:
    data_myTest = pickle.load(fp)
    
with open('data/labels_myTest.pkl', 'rb') as fp:
    labels_myTest = pickle.load(fp)

Data preprocessing and visualization¶

In [5]:
print("-- Soccer Events and real soccer events Dataset --")
print("[n.images, (width, height, depth)]")
for folder in (data_soccer):
    print(f"--- {folder} ---")
    print(f"Soccer shape: {len(data_soccer[folder])} {data_soccer[folder][1].shape}")
    print(f"Real soccer shape: {len(real_data_soccer[folder])} {real_data_soccer[folder][1].shape}")

soccer_sample = data_soccer['Cards'][22]
real_soccer_sample = real_data_soccer['Center'][1]

print("-"*30)

print(f"Soccer label: {labels_soccer[22]}")
print(f"Real soccer label: {real_labels_soccer[11]}")

plt.subplot(1,2,1),plt.imshow(soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.subplot(1,2,2),plt.imshow(real_soccer_sample, cmap='gray', vmin=0, vmax=255)
-- Soccer Events and real soccer events Dataset --
[n.images, (width, height, depth)]
--- Cards ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Center ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Corner ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Free-Kick ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Left ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Penalty ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Right ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- Tackle ---
Soccer shape: 6000 (80, 80)
Real soccer shape: 10 (80, 80)
--- To Substitue ---
Soccer shape: 5999 (80, 80)
Real soccer shape: 10 (80, 80)
------------------------------
Soccer label: Cards
Real soccer label: Center
Out[5]:
(<Axes: >, <matplotlib.image.AxesImage at 0x246e3f8feb0>)
No description has been provided for this image
In [6]:
print("-- Generic Events Dataset --")
print("[n.images, (width, height, depth)]")
for folder in (data_generic_events):
    print(f"--- {folder} ---")
    print(f"Soccer shape: {len(data_generic_events[folder])} {data_generic_events[folder][1].shape}")
    
generic_soccer_sample = data_generic_events['Soccer'][20]
random_image_sample = data_generic_events['Random'][20]

print("-"*30)

print(f"Label: {labels_generic_events[1420]}")
print(f"Label: {labels_generic_events[20]}")

plt.subplot(1,2,1),plt.imshow(generic_soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.subplot(1,2,2),plt.imshow(random_image_sample, cmap='gray', vmin=0, vmax=255)
-- Generic Events Dataset --
[n.images, (width, height, depth)]
--- Random ---
Soccer shape: 1400 (80, 80)
--- Soccer ---
Soccer shape: 1200 (80, 80)
------------------------------
Label: Soccer
Label: Random
Out[6]:
(<Axes: >, <matplotlib.image.AxesImage at 0x246e60dafd0>)
No description has been provided for this image
In [7]:
print("-- myTest Dataset --")
print("[n.images, (width, height, depth)]")
for folder in (data_myTest):
    print(f"--- {folder} ---")
    print(f"Soccer shape: {len(data_myTest[folder])} {data_myTest[folder][1].shape}")
    
generic_soccer_sample = data_myTest['Cards'][6]
random_image_sample = data_myTest['Events'][6]

print("-"*30)

print(f"Label: {labels_myTest[6]}")
print(f"Label: {labels_myTest[36]}")

plt.subplot(1,2,1),plt.imshow(generic_soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.subplot(1,2,2),plt.imshow(random_image_sample, cmap='gray', vmin=0, vmax=255)
-- myTest Dataset --
[n.images, (width, height, depth)]
--- Cards ---
Soccer shape: 10 (80, 80)
--- Center ---
Soccer shape: 10 (80, 80)
--- Corner ---
Soccer shape: 10 (80, 80)
--- Events ---
Soccer shape: 10 (80, 80)
--- Free-Kick ---
Soccer shape: 10 (80, 80)
--- Generic ---
Soccer shape: 10 (80, 80)
--- Left ---
Soccer shape: 10 (80, 80)
--- Penalty ---
Soccer shape: 10 (80, 80)
--- Right ---
Soccer shape: 10 (80, 80)
--- Tackle ---
Soccer shape: 10 (80, 80)
--- To Substitue ---
Soccer shape: 10 (80, 80)
------------------------------
Label: Cards
Label: Events
Out[7]:
(<Axes: >, <matplotlib.image.AxesImage at 0x246ee822f70>)
No description has been provided for this image

Label encoding¶

In [8]:
def one_hot_encoding(
    labels: pd.DataFrame,
    num_classes: int
) -> pd.DataFrame:
    """ Return the labels encoded with one-hot encoding.
    
    Parameters
    ------------------------
    labels: pd.DataFrame
        The labels to be one-hot encoded.
    num_classes: int
        The number of classes.

    Returns
    ------------------------
    pd.DataFrame: pd.DataFrame
        The one-hot encoded labels."""
    
    return pd.DataFrame(to_categorical(labels, num_classes))
In [9]:
n_generic_soccer_samples = labels_generic_events.count("Soccer")
In [10]:
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()

print("Soccer Events Labels")
labels_soccer = pd.DataFrame(le.fit_transform(labels_soccer))
labels_soccer = one_hot_encoding(labels_soccer, 9)

print(labels_soccer.sample(3))

print("-"*30)

print("Real Soccer Events Labels")
real_labels_soccer = pd.DataFrame(le.fit_transform(real_labels_soccer))
real_labels_soccer = one_hot_encoding(real_labels_soccer, 9)

print(real_labels_soccer.sample(3))

print("-"*30)
print("Soccer Events + Generic Labels")
labels_soccer_with_generic = labels_soccer.copy()

labels_soccer_with_generic[9] = 0.0

new_rows = pd.DataFrame([[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]] * n_generic_soccer_samples)
labels_soccer_with_generic = pd.concat([labels_soccer_with_generic, new_rows])

print(labels_soccer_with_generic.sample(3))

print("-"*30)

print("Generic Events Labels")
labels_generic_events = pd.DataFrame(le.fit_transform(labels_generic_events))
labels_generic_events = one_hot_encoding(labels_generic_events, 2)

print(labels_generic_events.sample(3))

print("-"*30)

print("my Test Labels")
labels_myTest = pd.DataFrame(le.fit_transform(labels_myTest))
labels_myTest = one_hot_encoding(labels_myTest, 11)

print(labels_myTest.sample(3))
Soccer Events Labels
         0    1    2    3    4    5    6    7    8
22536  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0
13976  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0
43885  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0
------------------------------
Real Soccer Events Labels
      0    1    2    3    4    5    6    7    8
71  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0
28  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0
48  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0
------------------------------
Soccer Events + Generic Labels
         0    1    2    3    4    5    6    7    8    9
12044  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0
46546  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0
7338   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0
------------------------------
Generic Events Labels
        0    1
1347  1.0  0.0
2438  0.0  1.0
2381  0.0  1.0
------------------------------
my Test Labels
     0    1    2    3    4    5    6    7    8    9    10
49  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0
94  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0
71  0.0  0.0  0.0  0.0  0.0  0.0  0.0  1.0  0.0  0.0  0.0

Data Augmentation¶

Function to perform data augmentation to expand the dataset size and improve generalization capabilities.

In [11]:
def augment_image(input_image):
    """ Return the augmented input image (randomly applies rotation, shear, zoom, crop, flip, brightness adjust)

    Parameters
    ------------------------
    input_image: np.array
        The input image.

    Returns
    ------------------------
    np.array: np.array
        The augmented image."""

    # Randomly rotate the image (between -30 and 30 degrees)
    height, width = input_image.shape[:2]
    angle = random.randint(-30, 30)
    rotation_matrix = cv2.getRotationMatrix2D((width / 2, height / 2), angle, 1)
    input_image = cv2.warpAffine(input_image, rotation_matrix, (width, height))
    
    # Apply shearing (0.15)
    shear_value = random.uniform(-0.15, 0.15)
    shear_matrix = np.array([[1.0, shear_value, 0.0], [0.0, 1.0, 0.0]])
    input_image = cv2.warpAffine(input_image, shear_matrix, (width, height))
    input_image = np.expand_dims(input_image, axis=-1)
    
    # Randomly zoom (scaling factor between 0.9 and 1.1)
    zoom_factor = tf.random.uniform(shape=[], minval=0.9, maxval=1.1)
    new_size = tf.cast(tf.shape(input_image)[:2], tf.float32) * zoom_factor
    input_image = tf.image.resize(input_image, tf.cast(new_size, tf.int32))

    # Randomly crop (90% of the original size)
    crop_size = tf.cast(tf.shape(input_image)[:2] * 0.9, tf.int32)
    input_image = tf.image.random_crop(input_image, size=[crop_size[0], crop_size[1], 1])
    
    # Randomly adjust brightness (by 0.2)
    input_image = tf.image.random_brightness(input_image, max_delta=0.2)
    
    # Reshape the augmented image
    input_image = np.array(input_image)
    input_image = cv2.resize(input_image, (soccer_images_dim,soccer_images_dim), interpolation=cv2.INTER_CUBIC)
    input_image = input_image.reshape(soccer_images_dim, soccer_images_dim, 1)
    
    return input_image
In [12]:
print("-- Just for explanatory purpose --")

# Augment the image
input_image = np.expand_dims(soccer_sample, axis=-1)
augmented_image = augment_image(input_image)

# Display the original and augmented images
plt.figure(figsize=(10, 5))
plt.subplot(1,2,1)
plt.title('Original Image')
plt.imshow(soccer_sample, cmap='gray', vmin=0, vmax=255)
plt.axis('off')

plt.subplot(1,2,2)
plt.title('Augmented Image')
plt.imshow(augmented_image, cmap='gray', vmin=0, vmax=255)
plt.axis('off')

plt.show()
-- Just for explanatory purpose --
No description has been provided for this image

Learning¶

The technique used to generate the holdouts for the evaluation of the various architectures is the stratified Monte-Carlo method, producing each time a different arrangement of the training and testing set, while keeping roughly the same class balance.

2 holdouts are considered to train and evaluate each model, 20% of the whole dataset is used for testing, 80% for training.

Below are also some functions to perform the training and evaluation of models.

In [13]:
number_of_splits = 2

holdouts_generator = StratifiedShuffleSplit(
    n_splits = number_of_splits,
    test_size = 0.2,
    random_state = 42
)
In [14]:
def train_model(
    model: tf.keras.Model,
    x_train: np.ndarray,
    x_test: np.ndarray,
    y_train: np.ndarray,
    y_test: np.ndarray,
    epochs: int,
    batch_size: int
):
    """ Train the model and return the performance metrics (accuracy, AUROC, AUPRC) for train and test and the training history.
    
    Parameters
    ------------------------
    model: tf.keras.Model
        The model to train.
    x_train: np.ndarray
        The input data for training.
    x_test: np.ndarray
        The input data for testing.
    y_train: np.ndarray
        The labels of the input data for training.
    y_test: np.ndarray
        The labels of the input data for testing.
    epochs: int
        The number of times the learning algorithm works through the dataset.
    batch_size: int
        The number of samples to work through before updating the model.

    Returns
    ------------------------
    The performance metrics and history of the model."""
    
    model_history = model.fit(
        x_train, y_train,
        validation_data = (x_test, y_test),
        epochs = epochs,
        batch_size = batch_size,
        callbacks = [EarlyStopping("val_loss", patience=2)]
    )

    train_evaluation = model.evaluate(x_train, y_train, return_dict=True)
    test_evaluation = model.evaluate(x_test, y_test, return_dict=True)
    
    metrics = {"train_evaluation": train_evaluation, "test_evaluation": test_evaluation}
    
    
    return metrics, model_history
In [15]:
def train_model_cp(
    model: tf.keras.Model,
    x_train: np.ndarray,
    x_test: np.ndarray,
    y_train: np.ndarray,
    y_test: np.ndarray,
    epochs: int,
    batch_size: int,
    cp: keras.callbacks.ModelCheckpoint
):
    """ Train the model with checkpoints and return the performance metrics (accuracy, AUROC, AUPRC) for train and test and the training history.
    
    Parameters
    ------------------------
    model: tf.keras.Model
        The model to train.
    x_train: np.ndarray
        The input data for training.
    x_test: np.ndarray
        The input data for testing.
    y_train: np.ndarray
        The labels of the input data for training.
    y_test: np.ndarray
        The labels of the input data for testing.
    epochs: int
        The number of times the learning algorithm works through the dataset.
    batch_size: int
        The number of samples to work through before updating the model.
    cp: keras.callbacks.ModelCheckpoint
        Checkpoint callback.

    Returns
    ------------------------
    The performance metrics and history of the model."""
    
    model_history = model.fit(
        x_train, y_train,
        validation_data = (x_test, y_test),
        epochs = epochs,
        batch_size = batch_size,
        callbacks = [EarlyStopping("val_loss", patience=4), cp]
    )

    #train_evaluation = model.evaluate(x_train, y_train, return_dict=True)
    test_evaluation = model.evaluate(x_test, y_test, return_dict=True)
    
    #metrics = {"train_evaluation": train_evaluation, "test_evaluation": test_evaluation}
    metrics = {"test_evaluation": test_evaluation}
    
    return metrics, model_history
In [16]:
def get_predictions(
    model: tf.keras.Model,
    x_test: np.ndarray,
    batch_size: int
):
    """ Return the predictions of the model over the test data.
    
    Parameters
    ------------------------
    model: tf.keras.Model
        The model to train.
    x_test: np.ndarray
        The input data for testing.
    batch_size: int
        The number of samples to work through before updating the model.

    Returns
    ------------------------
    The predictions of the model over the test data."""
    
    test_predictions = model.predict(
        x_test,
        batch_size = batch_size
    )
    
    return test_predictions

Evaluate a model¶

Next are a set of functions to evaluate and visualize the performances of models.

To have a statistically sound estimate of an architecture performance, multiple models are built and trained, each with the same architecture, over different portions of the data (holdouts) and, the average performance of those, is considered as an estimate of the overall performance of the architecture.

In [17]:
def model_metrics_holdout_estimate(
    model_metrics: list,
    holdout_total_number: int
) -> dict[str, float]:
    """ Return the average metrics (accuracy, AUROC, AUPRC) of the model accross all the holdouts for train and test.
    
    Parameters
    ------------------------
    model_metrics: list
        The list with the train/test metrics (accuracy, AUROC, AUPRC) for all the holdouts.
    holdout_total_number: int
        The number of total holdouts.

    Returns
    ------------------------
    dict: dict[str, float]
        The average metrics of the model accross all the holdouts for train and test."""
    
    total_accuracy_train = 0
    total_accuracy_test = 0
    total_AUROC_train = 0
    total_AUROC_test = 0
    total_AUPRC_train = 0
    total_AUPRC_test = 0

    for holdout in model_metrics:
        total_accuracy_train += holdout["train_evaluation"]['accuracy']
        total_AUROC_train += holdout["train_evaluation"]['AUROC']
        total_AUPRC_train += holdout["train_evaluation"]['AUPRC']

    for holdout in model_metrics:
        total_accuracy_test += holdout["test_evaluation"]['accuracy']
        total_AUROC_test += holdout["test_evaluation"]['AUROC']
        total_AUPRC_test += holdout["test_evaluation"]['AUPRC']

    avg_accuracy_train = total_accuracy_train / holdout_total_number
    avg_accuracy_test = total_accuracy_test / holdout_total_number
    avg_AUROC_train = total_AUROC_train / holdout_total_number
    avg_AUROC_test = total_AUROC_test / holdout_total_number
    avg_AUPRC_train = total_AUPRC_train / holdout_total_number
    avg_AUPRC_test = total_AUPRC_test / holdout_total_number

    return {
        'accuracy_train': avg_accuracy_train,
        'accuracy_test': avg_accuracy_test,
        'AUROC_train': avg_AUROC_train,
        'AUROC_test': avg_AUROC_test,
        'AUPRC_train': avg_AUPRC_train,
        'AUPRC_test': avg_AUPRC_test,
    }
In [18]:
def plot_train_history(model_history):
    """ Plots the metrics (accuracy, AUROC, AUPRC) of the model across the epochs and for each holdout.
    
    Parameters
    ------------------------
    model_history: history
        The training record of the model."""

    for i in range(len(model_history)):
        fig = plt.figure(figsize=(12.7,4.51))
        fig.suptitle(str(i+1) + " Holdout")
        
        # Accuracy
        plt.subplot(1,3,1)
        plt.plot(model_history[i].history['accuracy'])
        plt.plot(model_history[i].history['val_accuracy'])
        plt.title('Accuracy')
        plt.ylabel('accuracy')
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='upper left')
        
        # AUROC
        plt.subplot(1,3,2)
        plt.plot(model_history[i].history['AUROC'])
        plt.plot(model_history[i].history['val_AUROC'])
        plt.title('AUROC')
        plt.ylabel('AUROC')
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='upper left')
        
        # AUPRC
        plt.subplot(1,3,3)
        plt.plot(model_history[i].history['AUPRC'])
        plt.plot(model_history[i].history['val_AUPRC'])
        plt.title('AUPRC')
        plt.ylabel('AUPRC')
        plt.xlabel('epoch')
        plt.legend(['train', 'test'], loc='upper left')

        plt.subplots_adjust(wspace=0.3)
        plt.show()

CNN¶

A convolutional neural network consists a set of convolutional layers (convolution+maxPooling) and a set of dense layer. To avoid overfitting of the model dropout layers were added in the dense portion.

Here is the function to build the main model.

In [19]:
def build_soccer_CNN(
    input_shape: tuple
) -> tf.keras.Model:
    """ Returns the basic soccer CNN model.

    Parameters
    ------------------------
    input_shape: tuple
        The shape of the input.

    Returns
    ------------------------
    model: model
        The soccer model."""

    CNN = Sequential(name="Soccer_CNN")
    CNN.add(layers.Input(input_shape))
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 256,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 256,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),

    CNN.add(layers.Flatten())
    
    for i in range(2):
        CNN.add(layers.Dense(units = 128, activation='relu'))
        CNN.add(layers.Dropout(0.5))
        
    CNN.add(layers.Dense(9, activation='softmax'))
    
    CNN.compile(
        optimizer='adam',
        loss='categorical_crossentropy',
        metrics=get_minimal_multiclass_metrics()
    )
    
    CNN.summary()
    
    return CNN
In [20]:
#data_soccer is a dict containing ['events'...] so put together classes to evaluate
data_soccer_all = []
for event in data_soccer:
    for element in data_soccer[event]:
        data_soccer_all.append(element)

print(f"Data shape: {np.array(data_soccer_all).shape}")
print(f"Labels shape: {labels_soccer.shape}")
Data shape: (53999, 80, 80)
Labels shape: (53999, 9)
In [21]:
data_soccer_all = [np.expand_dims(x, axis=-1) for x in data_soccer_all]
In [22]:
epochs = 100
batch_size = 14

Main training loop - Soccer CNN¶

  • Normalize images' values in 0-1 range.
  • Build and train the model over the training set and test it over the test set for the different holdouts.
In [23]:
print("---- Soccer CNN ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_all, labels_soccer))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([data_soccer_all[x]/255 for x in train_indices]), np.array([data_soccer_all[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]

    # Build CNN
    CNN = build_soccer_CNN(x_train[0].shape)

    print("- Training model:\n")
    CNN_holdout_metrics, CNN_holdout_history = train_model(
        CNN,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size
    )
    
    predictions = get_predictions(CNN, x_test, batch_size=batch_size)
    CNN_holdout_predictions.append(predictions)
    CNN_holdout_test_labels.append(y_test)

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(CNN)
---- Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d (Conv2D)             (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d (MaxPooling2D  (None, 40, 40, 128)      0         
 )                                                               
                                                                 
 conv2d_1 (Conv2D)           (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_1 (MaxPooling  (None, 20, 20, 128)      0         
 2D)                                                             
                                                                 
 conv2d_2 (Conv2D)           (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_2 (MaxPooling  (None, 10, 10, 256)      0         
 2D)                                                             
                                                                 
 conv2d_3 (Conv2D)           (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_3 (MaxPooling  (None, 5, 5, 256)        0         
 2D)                                                             
                                                                 
 flatten (Flatten)           (None, 6400)              0         
                                                                 
 dense (Dense)               (None, 128)               819328    
                                                                 
 dropout (Dropout)           (None, 128)               0         
                                                                 
 dense_1 (Dense)             (None, 128)               16512     
                                                                 
 dropout_1 (Dropout)         (None, 128)               0         
                                                                 
 dense_2 (Dense)             (None, 9)                 1161      
                                                                 
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
3086/3086 [==============================] - 37s 10ms/step - loss: 1.2341 - accuracy: 0.5270 - recall: 0.3408 - precision: 0.7308 - AUROC: 0.9027 - AUPRC: 0.5983 - val_loss: 0.6373 - val_accuracy: 0.7908 - val_recall: 0.6710 - val_precision: 0.8773 - val_AUROC: 0.9761 - val_AUPRC: 0.8686
Epoch 2/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.7161 - accuracy: 0.7545 - recall: 0.6664 - precision: 0.8304 - AUROC: 0.9665 - AUPRC: 0.8345 - val_loss: 0.4194 - val_accuracy: 0.8614 - val_recall: 0.8125 - val_precision: 0.9001 - val_AUROC: 0.9884 - val_AUPRC: 0.9339
Epoch 3/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.5331 - accuracy: 0.8254 - recall: 0.7739 - precision: 0.8727 - AUROC: 0.9799 - AUPRC: 0.8999 - val_loss: 0.3611 - val_accuracy: 0.8840 - val_recall: 0.8471 - val_precision: 0.9158 - val_AUROC: 0.9907 - val_AUPRC: 0.9476
Epoch 4/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.4401 - accuracy: 0.8571 - recall: 0.8192 - precision: 0.8926 - AUROC: 0.9856 - AUPRC: 0.9282 - val_loss: 0.3377 - val_accuracy: 0.8894 - val_recall: 0.8622 - val_precision: 0.9161 - val_AUROC: 0.9913 - val_AUPRC: 0.9519
Epoch 5/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3766 - accuracy: 0.8792 - recall: 0.8504 - precision: 0.9079 - AUROC: 0.9884 - AUPRC: 0.9447 - val_loss: 0.3129 - val_accuracy: 0.9000 - val_recall: 0.8756 - val_precision: 0.9228 - val_AUROC: 0.9920 - val_AUPRC: 0.9592
Epoch 6/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3438 - accuracy: 0.8902 - recall: 0.8657 - precision: 0.9179 - AUROC: 0.9902 - AUPRC: 0.9531 - val_loss: 0.2792 - val_accuracy: 0.9126 - val_recall: 0.8922 - val_precision: 0.9308 - val_AUROC: 0.9934 - val_AUPRC: 0.9663
Epoch 7/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3024 - accuracy: 0.9041 - recall: 0.8832 - precision: 0.9254 - AUROC: 0.9920 - AUPRC: 0.9622 - val_loss: 0.2689 - val_accuracy: 0.9124 - val_recall: 0.8954 - val_precision: 0.9322 - val_AUROC: 0.9936 - val_AUPRC: 0.9680
Epoch 8/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2789 - accuracy: 0.9118 - recall: 0.8935 - precision: 0.9313 - AUROC: 0.9929 - AUPRC: 0.9669 - val_loss: 0.2710 - val_accuracy: 0.9140 - val_recall: 0.8984 - val_precision: 0.9311 - val_AUROC: 0.9935 - val_AUPRC: 0.9687
Epoch 9/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2614 - accuracy: 0.9195 - recall: 0.9030 - precision: 0.9377 - AUROC: 0.9934 - AUPRC: 0.9701 - val_loss: 0.2289 - val_accuracy: 0.9295 - val_recall: 0.9116 - val_precision: 0.9459 - val_AUROC: 0.9949 - val_AUPRC: 0.9760
Epoch 10/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.2399 - accuracy: 0.9256 - recall: 0.9103 - precision: 0.9429 - AUROC: 0.9941 - AUPRC: 0.9738 - val_loss: 0.2399 - val_accuracy: 0.9244 - val_recall: 0.9094 - val_precision: 0.9385 - val_AUROC: 0.9943 - val_AUPRC: 0.9741
Epoch 11/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2241 - accuracy: 0.9293 - recall: 0.9150 - precision: 0.9458 - AUROC: 0.9948 - AUPRC: 0.9768 - val_loss: 0.2313 - val_accuracy: 0.9267 - val_recall: 0.9128 - val_precision: 0.9420 - val_AUROC: 0.9947 - val_AUPRC: 0.9759
1350/1350 [==============================] - 8s 6ms/step - loss: 0.1007 - accuracy: 0.9684 - recall: 0.9568 - precision: 0.9774 - AUROC: 0.9992 - AUPRC: 0.9955
338/338 [==============================] - 2s 6ms/step - loss: 0.2313 - accuracy: 0.9267 - recall: 0.9128 - precision: 0.9420 - AUROC: 0.9947 - AUPRC: 0.9759
772/772 [==============================] - 2s 2ms/step
1it [06:01, 361.24s/it]
-- HOLDOUT 2
Model: "Soccer_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_4 (Conv2D)           (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_4 (MaxPooling  (None, 40, 40, 128)      0         
 2D)                                                             
                                                                 
 conv2d_5 (Conv2D)           (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_5 (MaxPooling  (None, 20, 20, 128)      0         
 2D)                                                             
                                                                 
 conv2d_6 (Conv2D)           (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_6 (MaxPooling  (None, 10, 10, 256)      0         
 2D)                                                             
                                                                 
 conv2d_7 (Conv2D)           (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_7 (MaxPooling  (None, 5, 5, 256)        0         
 2D)                                                             
                                                                 
 flatten_1 (Flatten)         (None, 6400)              0         
                                                                 
 dense_3 (Dense)             (None, 128)               819328    
                                                                 
 dropout_2 (Dropout)         (None, 128)               0         
                                                                 
 dense_4 (Dense)             (None, 128)               16512     
                                                                 
 dropout_3 (Dropout)         (None, 128)               0         
                                                                 
 dense_5 (Dense)             (None, 9)                 1161      
                                                                 
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
3086/3086 [==============================] - 32s 10ms/step - loss: 1.2249 - accuracy: 0.5262 - recall: 0.3379 - precision: 0.7239 - AUROC: 0.9049 - AUPRC: 0.5958 - val_loss: 0.6334 - val_accuracy: 0.7835 - val_recall: 0.7003 - val_precision: 0.8517 - val_AUROC: 0.9748 - val_AUPRC: 0.8658
Epoch 2/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.6799 - accuracy: 0.7666 - recall: 0.6848 - precision: 0.8333 - AUROC: 0.9695 - AUPRC: 0.8462 - val_loss: 0.4233 - val_accuracy: 0.8591 - val_recall: 0.8205 - val_precision: 0.8992 - val_AUROC: 0.9876 - val_AUPRC: 0.9317
Epoch 3/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.5093 - accuracy: 0.8307 - recall: 0.7814 - precision: 0.8756 - AUROC: 0.9815 - AUPRC: 0.9068 - val_loss: 0.3348 - val_accuracy: 0.8916 - val_recall: 0.8635 - val_precision: 0.9176 - val_AUROC: 0.9919 - val_AUPRC: 0.9546
Epoch 4/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.4266 - accuracy: 0.8610 - recall: 0.8244 - precision: 0.8955 - AUROC: 0.9860 - AUPRC: 0.9310 - val_loss: 0.3322 - val_accuracy: 0.8869 - val_recall: 0.8576 - val_precision: 0.9134 - val_AUROC: 0.9919 - val_AUPRC: 0.9551
Epoch 5/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.3735 - accuracy: 0.8776 - recall: 0.8491 - precision: 0.9074 - AUROC: 0.9891 - AUPRC: 0.9463 - val_loss: 0.3304 - val_accuracy: 0.8869 - val_recall: 0.8620 - val_precision: 0.9153 - val_AUROC: 0.9918 - val_AUPRC: 0.9566
Epoch 6/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.3303 - accuracy: 0.8930 - recall: 0.8681 - precision: 0.9172 - AUROC: 0.9907 - AUPRC: 0.9549 - val_loss: 0.2683 - val_accuracy: 0.9073 - val_recall: 0.8889 - val_precision: 0.9266 - val_AUROC: 0.9939 - val_AUPRC: 0.9683
Epoch 7/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.2991 - accuracy: 0.9032 - recall: 0.8831 - precision: 0.9255 - AUROC: 0.9920 - AUPRC: 0.9620 - val_loss: 0.2750 - val_accuracy: 0.9111 - val_recall: 0.8956 - val_precision: 0.9272 - val_AUROC: 0.9932 - val_AUPRC: 0.9665
Epoch 8/100
3086/3086 [==============================] - 31s 10ms/step - loss: 0.2719 - accuracy: 0.9126 - recall: 0.8934 - precision: 0.9327 - AUROC: 0.9930 - AUPRC: 0.9676 - val_loss: 0.2615 - val_accuracy: 0.9175 - val_recall: 0.9019 - val_precision: 0.9336 - val_AUROC: 0.9932 - val_AUPRC: 0.9687
Epoch 9/100
3086/3086 [==============================] - 30s 10ms/step - loss: 0.2618 - accuracy: 0.9163 - recall: 0.9001 - precision: 0.9348 - AUROC: 0.9935 - AUPRC: 0.9697 - val_loss: 0.2661 - val_accuracy: 0.9144 - val_recall: 0.8997 - val_precision: 0.9306 - val_AUROC: 0.9929 - val_AUPRC: 0.9676
Epoch 10/100
3086/3086 [==============================] - 32s 10ms/step - loss: 0.2378 - accuracy: 0.9241 - recall: 0.9101 - precision: 0.9406 - AUROC: 0.9944 - AUPRC: 0.9745 - val_loss: 0.2486 - val_accuracy: 0.9197 - val_recall: 0.9059 - val_precision: 0.9354 - val_AUROC: 0.9937 - val_AUPRC: 0.9722
Epoch 11/100
3086/3086 [==============================] - 34s 11ms/step - loss: 0.2211 - accuracy: 0.9309 - recall: 0.9170 - precision: 0.9449 - AUROC: 0.9948 - AUPRC: 0.9773 - val_loss: 0.2425 - val_accuracy: 0.9232 - val_recall: 0.9147 - val_precision: 0.9366 - val_AUROC: 0.9932 - val_AUPRC: 0.9725
Epoch 12/100
3086/3086 [==============================] - 33s 11ms/step - loss: 0.2205 - accuracy: 0.9322 - recall: 0.9189 - precision: 0.9476 - AUROC: 0.9948 - AUPRC: 0.9776 - val_loss: 0.2394 - val_accuracy: 0.9284 - val_recall: 0.9149 - val_precision: 0.9402 - val_AUROC: 0.9937 - val_AUPRC: 0.9730
Epoch 13/100
3086/3086 [==============================] - 34s 11ms/step - loss: 0.1942 - accuracy: 0.9396 - recall: 0.9273 - precision: 0.9526 - AUROC: 0.9956 - AUPRC: 0.9815 - val_loss: 0.2810 - val_accuracy: 0.9198 - val_recall: 0.9094 - val_precision: 0.9321 - val_AUROC: 0.9919 - val_AUPRC: 0.9678
Epoch 14/100
3086/3086 [==============================] - 33s 11ms/step - loss: 0.1886 - accuracy: 0.9414 - recall: 0.9299 - precision: 0.9540 - AUROC: 0.9957 - AUPRC: 0.9823 - val_loss: 0.2398 - val_accuracy: 0.9261 - val_recall: 0.9146 - val_precision: 0.9380 - val_AUROC: 0.9935 - val_AUPRC: 0.9736
1350/1350 [==============================] - 9s 6ms/step - loss: 0.0852 - accuracy: 0.9731 - recall: 0.9665 - precision: 0.9789 - AUROC: 0.9992 - AUPRC: 0.9960
338/338 [==============================] - 2s 6ms/step - loss: 0.2398 - accuracy: 0.9261 - recall: 0.9146 - precision: 0.9380 - AUROC: 0.9935 - AUPRC: 0.9736
772/772 [==============================] - 2s 3ms/step
2it [13:42, 411.15s/it]
In [24]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_metrics, number_of_splits)

print(f"CNN Soccer Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']}  --  test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']}  --  test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']}  --  test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN Soccer - Train history:")
plot_train_history(CNN_history)
print("-"*100)
CNN Soccer Metrics - 2-holdouts estimate:
Accuracy : train - 0.9707747995853424  --  test - 0.9263888895511627
AUROC : train - 0.9991893470287323  --  test - 0.9941038489341736
AUPRC : train - 0.9957385659217834  --  test - 0.9747623205184937
--------------------------------------------------------------------------------
CNN Soccer - Train history:
No description has been provided for this image
No description has been provided for this image
----------------------------------------------------------------------------------------------------
In [64]:
# Soccer CNN Metrics
CNN_metrics
Out[64]:
[{'train_evaluation': {'loss': 0.10073351860046387,
   'accuracy': 0.9684252142906189,
   'recall': 0.9568045735359192,
   'precision': 0.9774398803710938,
   'AUROC': 0.9991887807846069,
   'AUPRC': 0.9954752922058105},
  'test_evaluation': {'loss': 0.23127177357673645,
   'accuracy': 0.9266666769981384,
   'recall': 0.9127777814865112,
   'precision': 0.9419971108436584,
   'AUROC': 0.9946945905685425,
   'AUPRC': 0.9758864641189575}},
 {'train_evaluation': {'loss': 0.08517780154943466,
   'accuracy': 0.9731243848800659,
   'recall': 0.9664807319641113,
   'precision': 0.9788526296615601,
   'AUROC': 0.9991899132728577,
   'AUPRC': 0.9960018396377563},
  'test_evaluation': {'loss': 0.23984651267528534,
   'accuracy': 0.926111102104187,
   'recall': 0.9146296381950378,
   'precision': 0.9379925727844238,
   'AUROC': 0.9935131072998047,
   'AUPRC': 0.9736381769180298}}]
In [83]:
# Save results (save dictionary to basic_CNN_metrics.pkl file)
with open('results/basic_CNN_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
In [28]:
# Save Soccer CNN
soccer_CNN = CNN_models[0]
soccer_CNN.save_weights('modelWeights/soccerCNN_Model.h5')

Confusion Matrix - Soccer CNN¶

In [30]:
model_and_holdout_to_plot = 0

true_labels = tf.math.argmax(CNN_holdout_test_labels[model_and_holdout_to_plot], axis=1)
predicted_labels = tf.math.argmax(CNN_holdout_predictions[model_and_holdout_to_plot], axis=1)

confusion_matrix = tf.math.confusion_matrix(
    true_labels,
    predicted_labels,
    num_classes=9,
    weights=None,
    name=None
)

sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set(rc = {'figure.figsize':(10,10)})
plt.xlabel('Predicted Labels', fontsize=14)
plt.ylabel('True Labels', fontsize=14)
plt.yticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="13")
plt.xticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="13" )
plt.show()
No description has been provided for this image

Main training loop - Soccer CNN with Data Augmentation¶

  • Generate augmented images equal to 20% of the original test split size for training.
  • Normalize images' values in 0-1 range.
  • Build and train the model over the training set and test it over the test set for the different holdouts.
In [23]:
epochs = 100
batch_size = 10
In [24]:
augmentation_rate = 0.2
In [25]:
print("---- Soccer CNN with Data Augmentation ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_all, labels_soccer))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data
    x_train, x_test = np.array([data_soccer_all[x] for x in train_indices]), np.array([data_soccer_all[x] for x in test_indices])
    y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]

    ## Data Augmentation
    # Select specified % of elements
    n_data_to_augment = int(augmentation_rate * len(x_train))
    selected_indices = np.random.choice(len(x_train), n_data_to_augment, replace=False)
    selected_elements = x_train[selected_indices]
    selected_labels = y_train.values[selected_indices]

    # Apply data augmentation to the selected elements
    augmented_elements = np.array([augment_image(x) for x in selected_elements])

    # Insert the modified elements in the training data
    x_train = np.insert(x_train, selected_indices, augmented_elements, axis=0)
    y_train = pd.DataFrame(np.insert(y_train, selected_indices, selected_labels, axis=0))

    # Normalization
    x_train, x_test = np.array([x/255 for x in x_train]), np.array([x/255 for x in x_test])

    # Build CNN
    CNN = build_soccer_CNN(x_train[0].shape)

    print("- Training model:\n")
    CNN_holdout_metrics, CNN_holdout_history = train_model(
        CNN,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size
    )

    predictions = get_predictions(CNN, x_test, batch_size=batch_size)
    CNN_holdout_predictions.append(predictions)
    CNN_holdout_test_labels.append(y_test)

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)

    CNN_models.append(CNN)
---- Soccer CNN with Data Augmentation ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d (Conv2D)             (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d (MaxPooling2D  (None, 40, 40, 128)      0         
 )                                                               
                                                                 
 conv2d_1 (Conv2D)           (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_1 (MaxPooling  (None, 20, 20, 128)      0         
 2D)                                                             
                                                                 
 conv2d_2 (Conv2D)           (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_2 (MaxPooling  (None, 10, 10, 256)      0         
 2D)                                                             
                                                                 
 conv2d_3 (Conv2D)           (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_3 (MaxPooling  (None, 5, 5, 256)        0         
 2D)                                                             
                                                                 
 flatten (Flatten)           (None, 6400)              0         
                                                                 
 dense (Dense)               (None, 128)               819328    
                                                                 
 dropout (Dropout)           (None, 128)               0         
                                                                 
 dense_1 (Dense)             (None, 128)               16512     
                                                                 
 dropout_1 (Dropout)         (None, 128)               0         
                                                                 
 dense_2 (Dense)             (None, 9)                 1161      
                                                                 
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
5184/5184 [==============================] - 52s 8ms/step - loss: 1.2681 - accuracy: 0.5088 - recall: 0.3092 - precision: 0.7106 - AUROC: 0.8968 - AUPRC: 0.5710 - val_loss: 0.6507 - val_accuracy: 0.7681 - val_recall: 0.6743 - val_precision: 0.8434 - val_AUROC: 0.9750 - val_AUPRC: 0.8560
Epoch 2/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.8268 - accuracy: 0.7079 - recall: 0.6022 - precision: 0.8033 - AUROC: 0.9564 - AUPRC: 0.7908 - val_loss: 0.5008 - val_accuracy: 0.8249 - val_recall: 0.7672 - val_precision: 0.8745 - val_AUROC: 0.9838 - val_AUPRC: 0.9075
Epoch 3/100
5184/5184 [==============================] - 43s 8ms/step - loss: 0.6854 - accuracy: 0.7634 - recall: 0.6863 - precision: 0.8397 - AUROC: 0.9690 - AUPRC: 0.8498 - val_loss: 0.3980 - val_accuracy: 0.8682 - val_recall: 0.8281 - val_precision: 0.9031 - val_AUROC: 0.9887 - val_AUPRC: 0.9390
Epoch 4/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.6047 - accuracy: 0.7959 - recall: 0.7324 - precision: 0.8601 - AUROC: 0.9752 - AUPRC: 0.8804 - val_loss: 0.3494 - val_accuracy: 0.8860 - val_recall: 0.8483 - val_precision: 0.9198 - val_AUROC: 0.9914 - val_AUPRC: 0.9509
Epoch 5/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5459 - accuracy: 0.8187 - recall: 0.7638 - precision: 0.8741 - AUROC: 0.9795 - AUPRC: 0.9005 - val_loss: 0.3935 - val_accuracy: 0.8651 - val_recall: 0.8245 - val_precision: 0.8999 - val_AUROC: 0.9889 - val_AUPRC: 0.9406
Epoch 6/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4967 - accuracy: 0.8339 - recall: 0.7865 - precision: 0.8871 - AUROC: 0.9823 - AUPRC: 0.9148 - val_loss: 0.2773 - val_accuracy: 0.9030 - val_recall: 0.8824 - val_precision: 0.9262 - val_AUROC: 0.9938 - val_AUPRC: 0.9675
Epoch 7/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4635 - accuracy: 0.8478 - recall: 0.8041 - precision: 0.8942 - AUROC: 0.9845 - AUPRC: 0.9252 - val_loss: 0.2787 - val_accuracy: 0.9073 - val_recall: 0.8816 - val_precision: 0.9293 - val_AUROC: 0.9939 - val_AUPRC: 0.9672
Epoch 8/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4358 - accuracy: 0.8578 - recall: 0.8179 - precision: 0.9011 - AUROC: 0.9859 - AUPRC: 0.9330 - val_loss: 0.2683 - val_accuracy: 0.9134 - val_recall: 0.8938 - val_precision: 0.9328 - val_AUROC: 0.9937 - val_AUPRC: 0.9696
Epoch 9/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4161 - accuracy: 0.8647 - recall: 0.8265 - precision: 0.9060 - AUROC: 0.9871 - AUPRC: 0.9386 - val_loss: 0.2930 - val_accuracy: 0.9053 - val_recall: 0.8738 - val_precision: 0.9348 - val_AUROC: 0.9931 - val_AUPRC: 0.9652
Epoch 10/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.3978 - accuracy: 0.8711 - recall: 0.8333 - precision: 0.9108 - AUROC: 0.9879 - AUPRC: 0.9424 - val_loss: 0.2729 - val_accuracy: 0.9145 - val_recall: 0.8967 - val_precision: 0.9341 - val_AUROC: 0.9930 - val_AUPRC: 0.9679
1620/1620 [==============================] - 10s 6ms/step - loss: 0.2302 - accuracy: 0.9232 - recall: 0.8887 - precision: 0.9547 - AUROC: 0.9964 - AUPRC: 0.9791
338/338 [==============================] - 2s 6ms/step - loss: 0.2729 - accuracy: 0.9145 - recall: 0.8966 - precision: 0.9341 - AUROC: 0.9930 - AUPRC: 0.9679
1080/1080 [==============================] - 2s 2ms/step
1it [08:12, 492.79s/it]
-- HOLDOUT 2
Model: "Soccer_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_4 (Conv2D)           (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_4 (MaxPooling  (None, 40, 40, 128)      0         
 2D)                                                             
                                                                 
 conv2d_5 (Conv2D)           (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_5 (MaxPooling  (None, 20, 20, 128)      0         
 2D)                                                             
                                                                 
 conv2d_6 (Conv2D)           (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_6 (MaxPooling  (None, 10, 10, 256)      0         
 2D)                                                             
                                                                 
 conv2d_7 (Conv2D)           (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_7 (MaxPooling  (None, 5, 5, 256)        0         
 2D)                                                             
                                                                 
 flatten_1 (Flatten)         (None, 6400)              0         
                                                                 
 dense_3 (Dense)             (None, 128)               819328    
                                                                 
 dropout_2 (Dropout)         (None, 128)               0         
                                                                 
 dense_4 (Dense)             (None, 128)               16512     
                                                                 
 dropout_3 (Dropout)         (None, 128)               0         
                                                                 
 dense_5 (Dense)             (None, 9)                 1161      
                                                                 
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
5184/5184 [==============================] - 43s 8ms/step - loss: 1.4094 - accuracy: 0.4294 - recall: 0.2111 - precision: 0.6709 - AUROC: 0.8688 - AUPRC: 0.4881 - val_loss: 0.8121 - val_accuracy: 0.6980 - val_recall: 0.5565 - val_precision: 0.8086 - val_AUROC: 0.9599 - val_AUPRC: 0.7904
Epoch 2/100
5184/5184 [==============================] - 43s 8ms/step - loss: 0.8983 - accuracy: 0.6779 - recall: 0.5478 - precision: 0.7837 - AUROC: 0.9489 - AUPRC: 0.7544 - val_loss: 0.5515 - val_accuracy: 0.8088 - val_recall: 0.7489 - val_precision: 0.8619 - val_AUROC: 0.9804 - val_AUPRC: 0.8936
Epoch 3/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.7300 - accuracy: 0.7464 - recall: 0.6587 - precision: 0.8249 - AUROC: 0.9653 - AUPRC: 0.8315 - val_loss: 0.4336 - val_accuracy: 0.8559 - val_recall: 0.8017 - val_precision: 0.9024 - val_AUROC: 0.9884 - val_AUPRC: 0.9334
Epoch 4/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.6428 - accuracy: 0.7793 - recall: 0.7102 - precision: 0.8483 - AUROC: 0.9724 - AUPRC: 0.8645 - val_loss: 0.4014 - val_accuracy: 0.8676 - val_recall: 0.8164 - val_precision: 0.9057 - val_AUROC: 0.9889 - val_AUPRC: 0.9386
Epoch 5/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5860 - accuracy: 0.8023 - recall: 0.7401 - precision: 0.8632 - AUROC: 0.9767 - AUPRC: 0.8860 - val_loss: 0.3828 - val_accuracy: 0.8754 - val_recall: 0.8373 - val_precision: 0.9098 - val_AUROC: 0.9887 - val_AUPRC: 0.9421
Epoch 6/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5504 - accuracy: 0.8147 - recall: 0.7576 - precision: 0.8732 - AUROC: 0.9792 - AUPRC: 0.8979 - val_loss: 0.3977 - val_accuracy: 0.8641 - val_recall: 0.8244 - val_precision: 0.8995 - val_AUROC: 0.9885 - val_AUPRC: 0.9392
Epoch 7/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.5244 - accuracy: 0.8244 - recall: 0.7702 - precision: 0.8796 - AUROC: 0.9809 - AUPRC: 0.9068 - val_loss: 0.3704 - val_accuracy: 0.8731 - val_recall: 0.8331 - val_precision: 0.9118 - val_AUROC: 0.9897 - val_AUPRC: 0.9455
Epoch 8/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4926 - accuracy: 0.8361 - recall: 0.7870 - precision: 0.8882 - AUROC: 0.9828 - AUPRC: 0.9165 - val_loss: 0.3330 - val_accuracy: 0.8944 - val_recall: 0.8536 - val_precision: 0.9275 - val_AUROC: 0.9914 - val_AUPRC: 0.9545
Epoch 9/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4753 - accuracy: 0.8437 - recall: 0.7958 - precision: 0.8931 - AUROC: 0.9837 - AUPRC: 0.9213 - val_loss: 0.2953 - val_accuracy: 0.9052 - val_recall: 0.8709 - val_precision: 0.9316 - val_AUROC: 0.9933 - val_AUPRC: 0.9639
Epoch 10/100
5184/5184 [==============================] - 43s 8ms/step - loss: 0.4521 - accuracy: 0.8500 - recall: 0.8058 - precision: 0.8969 - AUROC: 0.9850 - AUPRC: 0.9278 - val_loss: 0.2753 - val_accuracy: 0.9100 - val_recall: 0.8869 - val_precision: 0.9328 - val_AUROC: 0.9935 - val_AUPRC: 0.9676
Epoch 11/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4338 - accuracy: 0.8580 - recall: 0.8160 - precision: 0.9029 - AUROC: 0.9860 - AUPRC: 0.9332 - val_loss: 0.2979 - val_accuracy: 0.9047 - val_recall: 0.8749 - val_precision: 0.9307 - val_AUROC: 0.9928 - val_AUPRC: 0.9634
Epoch 12/100
5184/5184 [==============================] - 42s 8ms/step - loss: 0.4214 - accuracy: 0.8626 - recall: 0.8217 - precision: 0.9068 - AUROC: 0.9867 - AUPRC: 0.9368 - val_loss: 0.3141 - val_accuracy: 0.8957 - val_recall: 0.8652 - val_precision: 0.9244 - val_AUROC: 0.9920 - val_AUPRC: 0.9587
1620/1620 [==============================] - 10s 6ms/step - loss: 0.2824 - accuracy: 0.9041 - recall: 0.8597 - precision: 0.9442 - AUROC: 0.9950 - AUPRC: 0.9700
338/338 [==============================] - 2s 6ms/step - loss: 0.3141 - accuracy: 0.8957 - recall: 0.8652 - precision: 0.9244 - AUROC: 0.9920 - AUPRC: 0.9587
1080/1080 [==============================] - 2s 2ms/step
2it [17:42, 531.15s/it]
In [26]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_metrics, number_of_splits)
print(f"CNN Soccer Data Augmentation - Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']}  --  test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']}  --  test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']}  --  test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN Soccer Data Augmentation - Train history:")
plot_train_history(CNN_history)
print("-"*100)
CNN Soccer Data Augmentation - Metrics - 2-holdouts estimate:
Accuracy : train - 0.9136347770690918  --  test - 0.9051388800144196
AUROC : train - 0.9957026243209839  --  test - 0.9924991726875305
AUPRC : train - 0.9745860695838928  --  test - 0.9633217751979828
--------------------------------------------------------------------------------
CNN Soccer Data Augmentation - Train history:
No description has been provided for this image
No description has been provided for this image
----------------------------------------------------------------------------------------------------
In [27]:
# Soccer CNN Data augmentation Metrics
CNN_metrics
Out[27]:
[{'train_evaluation': {'loss': 0.23024460673332214,
   'accuracy': 0.9232030510902405,
   'recall': 0.8886530995368958,
   'precision': 0.9546961784362793,
   'AUROC': 0.9964159727096558,
   'AUPRC': 0.9791415929794312},
  'test_evaluation': {'loss': 0.272890567779541,
   'accuracy': 0.9145370125770569,
   'recall': 0.896574079990387,
   'precision': 0.9341115355491638,
   'AUROC': 0.9929803609848022,
   'AUPRC': 0.9679373502731323}},
 {'train_evaluation': {'loss': 0.2823828458786011,
   'accuracy': 0.9040665030479431,
   'recall': 0.85965895652771,
   'precision': 0.9441913962364197,
   'AUROC': 0.994989275932312,
   'AUPRC': 0.9700305461883545},
  'test_evaluation': {'loss': 0.31406882405281067,
   'accuracy': 0.8957407474517822,
   'recall': 0.8651852011680603,
   'precision': 0.9244163036346436,
   'AUROC': 0.9920179843902588,
   'AUPRC': 0.9587062001228333}}]
In [85]:
# Save results (save dictionary to data_augmentation_basic_CNN_metrics.pkl file)
with open('results/data_augmentation_basic_CNN_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
In [28]:
# Save Soccer CNN with Data Augmentation
soccer_data_augmentation_CNN = CNN_models[0]
soccer_data_augmentation_CNN.save_weights('modelWeights/soccerModelDataAugmentation.h5')

Transfer Learning¶

Import ResNet50 and VGG16 architectures pretrained (imagenet weights) and add an average pooling layer as well as a dense layer with dropout.

  • ResNet50's final 3 layers are trained as well as the added ones.
  • VGG16 is instead kept fully frozen and only the added layers are trained.

The models require three channel images, thus they are tested on gray scale images reshaped to have three channels as well as RGB images.

In [23]:
# 3 channels gray input
threeChannels_input = [cv2.cvtColor(x, cv2.COLOR_GRAY2RGB) for x in data_soccer_all]
image_shape = threeChannels_input[0].shape
image_shape
Out[23]:
(80, 80, 3)
In [24]:
# RGB input

#data_soccer is a dict containing ['events'...] so put together classes to evaluate
data_soccer_rgb_all = []
for event in data_soccer_rgb:
    for element in data_soccer_rgb[event]:
        data_soccer_rgb_all.append(element)
print(f"RGB Data shape: {np.array(data_soccer_rgb_all).shape}")

rgb_input = data_soccer_rgb_all
image_shape = rgb_input[0].shape
image_shape
RGB Data shape: (53999, 80, 80, 3)
Out[24]:
(80, 80, 3)
In [25]:
# Create the base model from the pre-trained model MobileNet V2
resNet_model = tf.keras.applications.resnet.ResNet50(input_shape=image_shape,
                                               include_top=False,
                                               weights='imagenet')
In [26]:
# Create the base model from the pre-trained model VGG16
vgg16_model = tf.keras.applications.vgg16.VGG16(input_shape=image_shape,
                                               include_top=False,
                                               weights='imagenet')
In [27]:
# Freeze pretrained layers
resNet_model.trainable = False
vgg16_model.trainable = False
In [28]:
resNet_model.trainable = True

print("Number of layers in the base model: ", len(resNet_model.layers))

# Fine-tune from this layer onwards
fine_tune_at = 172

# Freeze all the layers before the `fine_tune_at` layer
for layer in resNet_model.layers[:fine_tune_at]:
  layer.trainable = False

print(f"Fine tuning from layer: {fine_tune_at}")
Number of layers in the base model:  175
Fine tuning from layer: 172
In [29]:
# Additional downstream layers on top of main model
avg_pooling = layers.GlobalAveragePooling2D()

dense1 = layers.Dense(units = 256, activation='relu')
dropout1 = layers.Dropout(0.5)

prediction_layer = layers.Dense(9, activation='softmax')
In [30]:
# Build final model ResNet
inputs = tf.keras.Input(shape=(soccer_images_dim, soccer_images_dim, 3))
x = resNet_model(inputs)
x = avg_pooling(x)
x = dense1(x)
x = dropout1(x)
outputs = prediction_layer(x)
resNet_soccer_model = tf.keras.Model(inputs, outputs)
resNet_soccer_model.summary()
Model: "model"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_3 (InputLayer)        [(None, 80, 80, 3)]       0         
                                                                 
 resnet50 (Functional)       (None, 3, 3, 2048)        23587712  
                                                                 
 global_average_pooling2d (G  (None, 2048)             0         
 lobalAveragePooling2D)                                          
                                                                 
 dense (Dense)               (None, 256)               524544    
                                                                 
 dropout (Dropout)           (None, 256)               0         
                                                                 
 dense_1 (Dense)             (None, 9)                 2313      
                                                                 
=================================================================
Total params: 24,114,569
Trainable params: 530,953
Non-trainable params: 23,583,616
_________________________________________________________________
In [31]:
resNet_soccer_model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=get_minimal_multiclass_metrics()
)

Main training loop - ResNet CNN (Gray and RGB images)¶

  • Normalize images' values in 0-1 range.
  • Train the unfrozen layers and additional ones of the ResNet model over the training set and test it over the test set for the different holdouts.
  • For memory and speed requirements these models are run over a single holdout and trained using checkpoints.
In [33]:
number_of_splits = 1

holdouts_generator = StratifiedShuffleSplit(
    n_splits = number_of_splits,
    test_size = 0.2,
    random_state = 42
)
In [34]:
epochs = 100
batch_size = 24
In [37]:
checkpoint_path = "training_ResNet_gray/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=True,
                                                 save_best_only=True,
                                                 verbose=1)
interrupt = False
enable_checkpoints = True
In [38]:
# Flag to load weights if training got interrupted
interrupt = True
In [39]:
print("---- ResNet Soccer CNN Gray ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(threeChannels_input, labels_soccer))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([threeChannels_input[x]/255 for x in train_indices]), np.array([threeChannels_input[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]

    # Get model
    model = resNet_soccer_model
    
    # Load weights if training got interrupted
    if interrupt:
        model.load_weights(checkpoint_path)

    CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
        model,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size,
        cp_callback
    )

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(model)
---- ResNet Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Epoch 1/100
5400/5400 [==============================] - ETA: 0s - loss: 0.7055 - accuracy: 0.7354 - recall: 0.6543 - precision: 0.8096 - AUROC: 0.9677 - AUPRC: 0.8299
Epoch 1: val_loss improved from inf to 0.64415, saving model to training_ResNet_gray\cp.ckpt
5400/5400 [==============================] - 85s 14ms/step - loss: 0.7055 - accuracy: 0.7354 - recall: 0.6543 - precision: 0.8096 - AUROC: 0.9677 - AUPRC: 0.8299 - val_loss: 0.6442 - val_accuracy: 0.7766 - val_recall: 0.7235 - val_precision: 0.8237 - val_AUROC: 0.9721 - val_AUPRC: 0.8570
Epoch 2/100
5396/5400 [============================>.] - ETA: 0s - loss: 0.7060 - accuracy: 0.7368 - recall: 0.6550 - precision: 0.8119 - AUROC: 0.9676 - AUPRC: 0.8302
Epoch 2: val_loss improved from 0.64415 to 0.63572, saving model to training_ResNet_gray\cp.ckpt
5400/5400 [==============================] - 71s 13ms/step - loss: 0.7061 - accuracy: 0.7367 - recall: 0.6550 - precision: 0.8119 - AUROC: 0.9676 - AUPRC: 0.8302 - val_loss: 0.6357 - val_accuracy: 0.7869 - val_recall: 0.7277 - val_precision: 0.8328 - val_AUROC: 0.9731 - val_AUPRC: 0.8615
Epoch 3/100
5397/5400 [============================>.] - ETA: 0s - loss: 0.7067 - accuracy: 0.7374 - recall: 0.6565 - precision: 0.8128 - AUROC: 0.9677 - AUPRC: 0.8303
Epoch 3: val_loss improved from 0.63572 to 0.62216, saving model to training_ResNet_gray\cp.ckpt
5400/5400 [==============================] - 72s 13ms/step - loss: 0.7067 - accuracy: 0.7374 - recall: 0.6565 - precision: 0.8128 - AUROC: 0.9677 - AUPRC: 0.8303 - val_loss: 0.6222 - val_accuracy: 0.7818 - val_recall: 0.7328 - val_precision: 0.8293 - val_AUROC: 0.9740 - val_AUPRC: 0.8662
Epoch 4/100
5397/5400 [============================>.] - ETA: 0s - loss: 0.7070 - accuracy: 0.7377 - recall: 0.6572 - precision: 0.8122 - AUROC: 0.9674 - AUPRC: 0.8299
Epoch 4: val_loss did not improve from 0.62216
5400/5400 [==============================] - 67s 12ms/step - loss: 0.7071 - accuracy: 0.7376 - recall: 0.6571 - precision: 0.8122 - AUROC: 0.9674 - AUPRC: 0.8299 - val_loss: 0.6446 - val_accuracy: 0.7794 - val_recall: 0.7085 - val_precision: 0.8388 - val_AUROC: 0.9728 - val_AUPRC: 0.8582
Epoch 5/100
5397/5400 [============================>.] - ETA: 0s - loss: 0.6991 - accuracy: 0.7382 - recall: 0.6585 - precision: 0.8125 - AUROC: 0.9683 - AUPRC: 0.8321
Epoch 5: val_loss improved from 0.62216 to 0.60718, saving model to training_ResNet_gray\cp.ckpt
5400/5400 [==============================] - 66s 12ms/step - loss: 0.6991 - accuracy: 0.7382 - recall: 0.6586 - precision: 0.8125 - AUROC: 0.9683 - AUPRC: 0.8321 - val_loss: 0.6072 - val_accuracy: 0.7924 - val_recall: 0.7368 - val_precision: 0.8367 - val_AUROC: 0.9752 - val_AUPRC: 0.8709
Epoch 6/100
5398/5400 [============================>.] - ETA: 0s - loss: 0.7042 - accuracy: 0.7347 - recall: 0.6549 - precision: 0.8095 - AUROC: 0.9678 - AUPRC: 0.8300
Epoch 6: val_loss did not improve from 0.60718
5400/5400 [==============================] - 68s 13ms/step - loss: 0.7044 - accuracy: 0.7347 - recall: 0.6549 - precision: 0.8095 - AUROC: 0.9678 - AUPRC: 0.8299 - val_loss: 0.6132 - val_accuracy: 0.7903 - val_recall: 0.7395 - val_precision: 0.8333 - val_AUROC: 0.9750 - val_AUPRC: 0.8715
Epoch 7/100
5400/5400 [==============================] - ETA: 0s - loss: 0.6988 - accuracy: 0.7410 - recall: 0.6609 - precision: 0.8141 - AUROC: 0.9683 - AUPRC: 0.8325
Epoch 7: val_loss did not improve from 0.60718
5400/5400 [==============================] - 70s 13ms/step - loss: 0.6988 - accuracy: 0.7410 - recall: 0.6609 - precision: 0.8141 - AUROC: 0.9683 - AUPRC: 0.8325 - val_loss: 0.6257 - val_accuracy: 0.7849 - val_recall: 0.7348 - val_precision: 0.8291 - val_AUROC: 0.9736 - val_AUPRC: 0.8631
338/338 [==============================] - 5s 13ms/step - loss: 0.6257 - accuracy: 0.7856 - recall: 0.7341 - precision: 0.8293 - AUROC: 0.9736 - AUPRC: 0.8633
1it [09:04, 544.35s/it]
In [40]:
print(f"ResNet Soccer CNN Gray Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate:
Accuracy : test - 0.7855555415153503
AUROC : test - 0.9735950231552124
AUPRC : test - 0.8632519245147705
--------------------------------------------------------------------------------
In [42]:
# CNN ResNet - Gray images - Metrics
CNN_metrics
Out[42]:
[{'test_evaluation': {'loss': 0.6256866455078125,
   'accuracy': 0.7855555415153503,
   'recall': 0.734074056148529,
   'precision': 0.8292887210845947,
   'AUROC': 0.9735950231552124,
   'AUPRC': 0.8632519245147705}}]
In [71]:
# Save results (save dictionary to resNet_gray_metrics.pkl file)
with open('results/resNet_gray_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
In [44]:
# Save Gray ResNet Soccer CNN
resNet_soccer_gray = CNN_models[0]
resNet_soccer_gray.save_weights('modelWeights/resNet_soccer_gray_model.h5')
In [35]:
checkpoint_path = "training_ResNet_rgb/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=True,
                                                 save_best_only=True,
                                                 verbose=1)
interrupt = False
In [36]:
# Flag to load weights if training got interrupted
interrupt = True
In [37]:
print("---- ResNet Soccer CNN RGB ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(rgb_input, labels_soccer))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([rgb_input[x]/255 for x in train_indices]), np.array([rgb_input[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]

    # Get model
    model = resNet_soccer_model
    
    # Load weights if training got interrupted
    if interrupt:
        model.load_weights(checkpoint_path)

    CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
        model,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size,
        cp_callback
    )
    
    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(model)
---- ResNet Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Epoch 1/100
1800/1800 [==============================] - ETA: 0s - loss: 0.8205 - accuracy: 0.6941 - recall: 0.5868 - precision: 0.7885 - AUROC: 0.9570 - AUPRC: 0.7813
Epoch 1: val_loss improved from inf to 0.69696, saving model to training_ResNet_rgb\cp.ckpt
1800/1800 [==============================] - 44s 19ms/step - loss: 0.8205 - accuracy: 0.6941 - recall: 0.5868 - precision: 0.7885 - AUROC: 0.9570 - AUPRC: 0.7813 - val_loss: 0.6970 - val_accuracy: 0.7419 - val_recall: 0.6651 - val_precision: 0.8115 - val_AUROC: 0.9685 - val_AUPRC: 0.8311
Epoch 2/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.8157 - accuracy: 0.6990 - recall: 0.5889 - precision: 0.7887 - AUROC: 0.9575 - AUPRC: 0.7829
Epoch 2: val_loss improved from 0.69696 to 0.67831, saving model to training_ResNet_rgb\cp.ckpt
1800/1800 [==============================] - 30s 17ms/step - loss: 0.8156 - accuracy: 0.6991 - recall: 0.5890 - precision: 0.7888 - AUROC: 0.9575 - AUPRC: 0.7830 - val_loss: 0.6783 - val_accuracy: 0.7570 - val_recall: 0.6832 - val_precision: 0.8214 - val_AUROC: 0.9699 - val_AUPRC: 0.8398
Epoch 3/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.8062 - accuracy: 0.7009 - recall: 0.5966 - precision: 0.7927 - AUROC: 0.9584 - AUPRC: 0.7876
Epoch 3: val_loss improved from 0.67831 to 0.65387, saving model to training_ResNet_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.8058 - accuracy: 0.7011 - recall: 0.5968 - precision: 0.7928 - AUROC: 0.9584 - AUPRC: 0.7878 - val_loss: 0.6539 - val_accuracy: 0.7609 - val_recall: 0.6978 - val_precision: 0.8247 - val_AUROC: 0.9719 - val_AUPRC: 0.8502
Epoch 4/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.8027 - accuracy: 0.7031 - recall: 0.5989 - precision: 0.7911 - AUROC: 0.9587 - AUPRC: 0.7886
Epoch 4: val_loss did not improve from 0.65387
1800/1800 [==============================] - 28s 16ms/step - loss: 0.8026 - accuracy: 0.7031 - recall: 0.5988 - precision: 0.7910 - AUROC: 0.9587 - AUPRC: 0.7885 - val_loss: 0.6727 - val_accuracy: 0.7613 - val_recall: 0.6639 - val_precision: 0.8455 - val_AUROC: 0.9717 - val_AUPRC: 0.8477
Epoch 5/100
1800/1800 [==============================] - ETA: 0s - loss: 0.7865 - accuracy: 0.7112 - recall: 0.6095 - precision: 0.7949 - AUROC: 0.9604 - AUPRC: 0.7956
Epoch 5: val_loss did not improve from 0.65387
1800/1800 [==============================] - 28s 16ms/step - loss: 0.7865 - accuracy: 0.7112 - recall: 0.6095 - precision: 0.7949 - AUROC: 0.9604 - AUPRC: 0.7956 - val_loss: 0.6634 - val_accuracy: 0.7576 - val_recall: 0.6902 - val_precision: 0.8189 - val_AUROC: 0.9711 - val_AUPRC: 0.8446
338/338 [==============================] - 5s 13ms/step - loss: 0.6637 - accuracy: 0.7575 - recall: 0.6895 - precision: 0.8184 - AUROC: 0.9711 - AUPRC: 0.8445
1it [03:22, 202.54s/it]
In [38]:
print(f"ResNet Soccer CNN RGB Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate:
Accuracy : test - 0.7574999928474426
AUROC : test - 0.9711117744445801
AUPRC : test - 0.8445291519165039
--------------------------------------------------------------------------------
In [39]:
# CNN ResNet - RGB images - Metrics
CNN_metrics
Out[39]:
[{'test_evaluation': {'loss': 0.663693368434906,
   'accuracy': 0.7574999928474426,
   'recall': 0.6895370483398438,
   'precision': 0.8184415698051453,
   'AUROC': 0.9711117744445801,
   'AUPRC': 0.8445291519165039}}]
In [73]:
# Save results (save dictionary to resNet_rgb_metrics.pkl file)
with open('results/resNet_rgb_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
In [40]:
# Save RGB ResNet Soccer CNN
resNet_soccer_rgb = CNN_models[0]
resNet_soccer_rgb.save_weights('modelWeights/resNet_soccer_rgb_model.h5')

Main training loop - VGG16 CNN (Gray and RGB images)¶

  • Normalize images' values in 0-1 range.
  • Train the additional layers of the VGG16 model over the training set and test it over the test set for the different holdouts.
In [32]:
vgg16_model.trainable = True

print("Number of layers in the base model: ", len(vgg16_model.layers))

# Fine-tune from this layer onwards
fine_tune_at = 19

# Freeze all the layers before the `fine_tune_at` layer
for layer in vgg16_model.layers[:fine_tune_at]:
  layer.trainable = False
Number of layers in the base model:  19
In [33]:
vgg16_model.summary()
Model: "vgg16"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_2 (InputLayer)        [(None, 80, 80, 3)]       0         
                                                                 
 block1_conv1 (Conv2D)       (None, 80, 80, 64)        1792      
                                                                 
 block1_conv2 (Conv2D)       (None, 80, 80, 64)        36928     
                                                                 
 block1_pool (MaxPooling2D)  (None, 40, 40, 64)        0         
                                                                 
 block2_conv1 (Conv2D)       (None, 40, 40, 128)       73856     
                                                                 
 block2_conv2 (Conv2D)       (None, 40, 40, 128)       147584    
                                                                 
 block2_pool (MaxPooling2D)  (None, 20, 20, 128)       0         
                                                                 
 block3_conv1 (Conv2D)       (None, 20, 20, 256)       295168    
                                                                 
 block3_conv2 (Conv2D)       (None, 20, 20, 256)       590080    
                                                                 
 block3_conv3 (Conv2D)       (None, 20, 20, 256)       590080    
                                                                 
 block3_pool (MaxPooling2D)  (None, 10, 10, 256)       0         
                                                                 
 block4_conv1 (Conv2D)       (None, 10, 10, 512)       1180160   
                                                                 
 block4_conv2 (Conv2D)       (None, 10, 10, 512)       2359808   
                                                                 
 block4_conv3 (Conv2D)       (None, 10, 10, 512)       2359808   
                                                                 
 block4_pool (MaxPooling2D)  (None, 5, 5, 512)         0         
                                                                 
 block5_conv1 (Conv2D)       (None, 5, 5, 512)         2359808   
                                                                 
 block5_conv2 (Conv2D)       (None, 5, 5, 512)         2359808   
                                                                 
 block5_conv3 (Conv2D)       (None, 5, 5, 512)         2359808   
                                                                 
 block5_pool (MaxPooling2D)  (None, 2, 2, 512)         0         
                                                                 
=================================================================
Total params: 14,714,688
Trainable params: 0
Non-trainable params: 14,714,688
_________________________________________________________________
In [34]:
# Additional downstream layers on top of main model
avg_pooling = layers.GlobalAveragePooling2D()
flattening = layers.Flatten()

dense1 = layers.Dense(units = 256, activation='relu')
dropout1 = layers.Dropout(0.5)
dense2 = layers.Dense(units = 128, activation='relu')
dropout2 = layers.Dropout(0.5)

prediction_layer = layers.Dense(9, activation='softmax')
In [35]:
# Build final model VGG16
inputs = tf.keras.Input(shape=(soccer_images_dim, soccer_images_dim, 3))
x = vgg16_model(inputs, training=False)
x = avg_pooling(x)
x = dense1(x)
x = dropout1(x)
outputs = prediction_layer(x)
vgg16_soccer_model = tf.keras.Model(inputs, outputs)
vgg16_soccer_model.summary()
Model: "model_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 input_4 (InputLayer)        [(None, 80, 80, 3)]       0         
                                                                 
 vgg16 (Functional)          (None, 2, 2, 512)         14714688  
                                                                 
 global_average_pooling2d_1   (None, 512)              0         
 (GlobalAveragePooling2D)                                        
                                                                 
 dense_2 (Dense)             (None, 256)               131328    
                                                                 
 dropout_1 (Dropout)         (None, 256)               0         
                                                                 
 dense_4 (Dense)             (None, 9)                 2313      
                                                                 
=================================================================
Total params: 14,848,329
Trainable params: 133,641
Non-trainable params: 14,714,688
_________________________________________________________________
In [36]:
vgg16_soccer_model.compile(
    optimizer='adam',
    loss='categorical_crossentropy',
    metrics=get_minimal_multiclass_metrics()
)

Main training loop - VGG16 CNN (Gray and RGB images)¶

  • Normalize images' values in 0-1 range.
  • Train the unfrozen layers and additional ones of the VGG16 model over the training set and test it over the test set for the different holdouts.
In [41]:
checkpoint_path = "training_vgg_gray/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=True,
                                                 save_best_only=True,
                                                 verbose=1)
interrupt = False
In [ ]:
# Flag to load weights if training got interrupted
interrupt = True
In [42]:
print("---- VGG16 Soccer CNN Gray ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(threeChannels_input, labels_soccer))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([threeChannels_input[x]/255 for x in train_indices]), np.array([threeChannels_input[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]

    # Get model
    model = vgg16_soccer_model
    
    # Load weights if training got interrupted
    if interrupt:
        model.load_weights(checkpoint_path)

    CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
        model,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size,
        cp_callback
    )

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(model)
---- VGG16 Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Epoch 1/100
1800/1800 [==============================] - ETA: 0s - loss: 0.9374 - accuracy: 0.6603 - recall: 0.5088 - precision: 0.7808 - AUROC: 0.9453 - AUPRC: 0.7365
Epoch 1: val_loss improved from inf to 0.67921, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 41s 18ms/step - loss: 0.9374 - accuracy: 0.6603 - recall: 0.5088 - precision: 0.7808 - AUROC: 0.9453 - AUPRC: 0.7365 - val_loss: 0.6792 - val_accuracy: 0.7593 - val_recall: 0.6690 - val_precision: 0.8331 - val_AUROC: 0.9710 - val_AUPRC: 0.8434
Epoch 2/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.7039 - accuracy: 0.7480 - recall: 0.6654 - precision: 0.8144 - AUROC: 0.9681 - AUPRC: 0.8307
Epoch 2: val_loss improved from 0.67921 to 0.62108, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.7041 - accuracy: 0.7479 - recall: 0.6653 - precision: 0.8143 - AUROC: 0.9681 - AUPRC: 0.8306 - val_loss: 0.6211 - val_accuracy: 0.7749 - val_recall: 0.6942 - val_precision: 0.8382 - val_AUROC: 0.9754 - val_AUPRC: 0.8648
Epoch 3/100
1800/1800 [==============================] - ETA: 0s - loss: 0.6365 - accuracy: 0.7702 - recall: 0.7033 - precision: 0.8265 - AUROC: 0.9737 - AUPRC: 0.8559
Epoch 3: val_loss improved from 0.62108 to 0.56647, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.6365 - accuracy: 0.7702 - recall: 0.7033 - precision: 0.8265 - AUROC: 0.9737 - AUPRC: 0.8559 - val_loss: 0.5665 - val_accuracy: 0.7948 - val_recall: 0.7282 - val_precision: 0.8531 - val_AUROC: 0.9789 - val_AUPRC: 0.8826
Epoch 4/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.5948 - accuracy: 0.7859 - recall: 0.7279 - precision: 0.8353 - AUROC: 0.9763 - AUPRC: 0.8707
Epoch 4: val_loss improved from 0.56647 to 0.54031, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5948 - accuracy: 0.7859 - recall: 0.7280 - precision: 0.8352 - AUROC: 0.9763 - AUPRC: 0.8707 - val_loss: 0.5403 - val_accuracy: 0.8046 - val_recall: 0.7635 - val_precision: 0.8444 - val_AUROC: 0.9802 - val_AUPRC: 0.8899
Epoch 5/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.5660 - accuracy: 0.7943 - recall: 0.7433 - precision: 0.8426 - AUROC: 0.9785 - AUPRC: 0.8806
Epoch 5: val_loss improved from 0.54031 to 0.49853, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5660 - accuracy: 0.7943 - recall: 0.7433 - precision: 0.8426 - AUROC: 0.9785 - AUPRC: 0.8806 - val_loss: 0.4985 - val_accuracy: 0.8217 - val_recall: 0.7799 - val_precision: 0.8611 - val_AUROC: 0.9830 - val_AUPRC: 0.9055
Epoch 6/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.5360 - accuracy: 0.8064 - recall: 0.7572 - precision: 0.8487 - AUROC: 0.9805 - AUPRC: 0.8913
Epoch 6: val_loss improved from 0.49853 to 0.48654, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5359 - accuracy: 0.8064 - recall: 0.7573 - precision: 0.8487 - AUROC: 0.9805 - AUPRC: 0.8913 - val_loss: 0.4865 - val_accuracy: 0.8242 - val_recall: 0.7819 - val_precision: 0.8603 - val_AUROC: 0.9836 - val_AUPRC: 0.9082
Epoch 7/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.5176 - accuracy: 0.8114 - recall: 0.7657 - precision: 0.8527 - AUROC: 0.9817 - AUPRC: 0.8979
Epoch 7: val_loss improved from 0.48654 to 0.46898, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5177 - accuracy: 0.8114 - recall: 0.7657 - precision: 0.8526 - AUROC: 0.9817 - AUPRC: 0.8979 - val_loss: 0.4690 - val_accuracy: 0.8329 - val_recall: 0.7958 - val_precision: 0.8635 - val_AUROC: 0.9846 - val_AUPRC: 0.9130
Epoch 8/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.5019 - accuracy: 0.8185 - recall: 0.7752 - precision: 0.8575 - AUROC: 0.9828 - AUPRC: 0.9029
Epoch 8: val_loss improved from 0.46898 to 0.46023, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5019 - accuracy: 0.8185 - recall: 0.7752 - precision: 0.8575 - AUROC: 0.9828 - AUPRC: 0.9029 - val_loss: 0.4602 - val_accuracy: 0.8369 - val_recall: 0.7983 - val_precision: 0.8700 - val_AUROC: 0.9848 - val_AUPRC: 0.9165
Epoch 9/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.4862 - accuracy: 0.8229 - recall: 0.7818 - precision: 0.8607 - AUROC: 0.9838 - AUPRC: 0.9081
Epoch 9: val_loss improved from 0.46023 to 0.45084, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4861 - accuracy: 0.8230 - recall: 0.7819 - precision: 0.8607 - AUROC: 0.9838 - AUPRC: 0.9081 - val_loss: 0.4508 - val_accuracy: 0.8375 - val_recall: 0.8064 - val_precision: 0.8699 - val_AUROC: 0.9856 - val_AUPRC: 0.9198
Epoch 10/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4745 - accuracy: 0.8261 - recall: 0.7883 - precision: 0.8622 - AUROC: 0.9844 - AUPRC: 0.9119
Epoch 10: val_loss improved from 0.45084 to 0.44901, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4745 - accuracy: 0.8261 - recall: 0.7883 - precision: 0.8622 - AUROC: 0.9844 - AUPRC: 0.9119 - val_loss: 0.4490 - val_accuracy: 0.8398 - val_recall: 0.8027 - val_precision: 0.8737 - val_AUROC: 0.9855 - val_AUPRC: 0.9199
Epoch 11/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4621 - accuracy: 0.8312 - recall: 0.7958 - precision: 0.8659 - AUROC: 0.9851 - AUPRC: 0.9153
Epoch 11: val_loss improved from 0.44901 to 0.44684, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 28s 16ms/step - loss: 0.4621 - accuracy: 0.8312 - recall: 0.7958 - precision: 0.8659 - AUROC: 0.9851 - AUPRC: 0.9153 - val_loss: 0.4468 - val_accuracy: 0.8400 - val_recall: 0.8036 - val_precision: 0.8733 - val_AUROC: 0.9857 - val_AUPRC: 0.9211
Epoch 12/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4504 - accuracy: 0.8352 - recall: 0.8000 - precision: 0.8670 - AUROC: 0.9858 - AUPRC: 0.9193
Epoch 12: val_loss improved from 0.44684 to 0.44189, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4504 - accuracy: 0.8352 - recall: 0.8000 - precision: 0.8670 - AUROC: 0.9858 - AUPRC: 0.9193 - val_loss: 0.4419 - val_accuracy: 0.8422 - val_recall: 0.8119 - val_precision: 0.8738 - val_AUROC: 0.9860 - val_AUPRC: 0.9213
Epoch 13/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.4418 - accuracy: 0.8390 - recall: 0.8054 - precision: 0.8700 - AUROC: 0.9863 - AUPRC: 0.9217
Epoch 13: val_loss improved from 0.44189 to 0.42453, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4419 - accuracy: 0.8390 - recall: 0.8054 - precision: 0.8700 - AUROC: 0.9863 - AUPRC: 0.9216 - val_loss: 0.4245 - val_accuracy: 0.8494 - val_recall: 0.8214 - val_precision: 0.8761 - val_AUROC: 0.9865 - val_AUPRC: 0.9271
Epoch 14/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.4351 - accuracy: 0.8402 - recall: 0.8080 - precision: 0.8720 - AUROC: 0.9867 - AUPRC: 0.9236
Epoch 14: val_loss improved from 0.42453 to 0.42399, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4351 - accuracy: 0.8402 - recall: 0.8080 - precision: 0.8720 - AUROC: 0.9867 - AUPRC: 0.9236 - val_loss: 0.4240 - val_accuracy: 0.8468 - val_recall: 0.8141 - val_precision: 0.8763 - val_AUROC: 0.9873 - val_AUPRC: 0.9278
Epoch 15/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.4281 - accuracy: 0.8439 - recall: 0.8112 - precision: 0.8733 - AUROC: 0.9870 - AUPRC: 0.9257
Epoch 15: val_loss improved from 0.42399 to 0.41646, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4282 - accuracy: 0.8438 - recall: 0.8112 - precision: 0.8732 - AUROC: 0.9870 - AUPRC: 0.9257 - val_loss: 0.4165 - val_accuracy: 0.8502 - val_recall: 0.8215 - val_precision: 0.8822 - val_AUROC: 0.9874 - val_AUPRC: 0.9299
Epoch 16/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4174 - accuracy: 0.8464 - recall: 0.8161 - precision: 0.8754 - AUROC: 0.9877 - AUPRC: 0.9289
Epoch 16: val_loss did not improve from 0.41646
1800/1800 [==============================] - 28s 15ms/step - loss: 0.4174 - accuracy: 0.8464 - recall: 0.8161 - precision: 0.8754 - AUROC: 0.9877 - AUPRC: 0.9289 - val_loss: 0.4189 - val_accuracy: 0.8547 - val_recall: 0.8271 - val_precision: 0.8768 - val_AUROC: 0.9870 - val_AUPRC: 0.9290
Epoch 17/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4108 - accuracy: 0.8499 - recall: 0.8216 - precision: 0.8779 - AUROC: 0.9880 - AUPRC: 0.9309
Epoch 17: val_loss improved from 0.41646 to 0.40520, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 28s 16ms/step - loss: 0.4108 - accuracy: 0.8499 - recall: 0.8216 - precision: 0.8779 - AUROC: 0.9880 - AUPRC: 0.9309 - val_loss: 0.4052 - val_accuracy: 0.8600 - val_recall: 0.8352 - val_precision: 0.8850 - val_AUROC: 0.9875 - val_AUPRC: 0.9338
Epoch 18/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.4051 - accuracy: 0.8507 - recall: 0.8225 - precision: 0.8788 - AUROC: 0.9883 - AUPRC: 0.9320
Epoch 18: val_loss improved from 0.40520 to 0.39420, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 28s 16ms/step - loss: 0.4050 - accuracy: 0.8508 - recall: 0.8226 - precision: 0.8788 - AUROC: 0.9883 - AUPRC: 0.9320 - val_loss: 0.3942 - val_accuracy: 0.8612 - val_recall: 0.8418 - val_precision: 0.8815 - val_AUROC: 0.9883 - val_AUPRC: 0.9368
Epoch 19/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3985 - accuracy: 0.8532 - recall: 0.8267 - precision: 0.8788 - AUROC: 0.9886 - AUPRC: 0.9342
Epoch 19: val_loss did not improve from 0.39420
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3985 - accuracy: 0.8532 - recall: 0.8267 - precision: 0.8788 - AUROC: 0.9886 - AUPRC: 0.9342 - val_loss: 0.4075 - val_accuracy: 0.8581 - val_recall: 0.8374 - val_precision: 0.8787 - val_AUROC: 0.9872 - val_AUPRC: 0.9323
Epoch 20/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.3862 - accuracy: 0.8566 - recall: 0.8302 - precision: 0.8842 - AUROC: 0.9893 - AUPRC: 0.9374
Epoch 20: val_loss did not improve from 0.39420
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3864 - accuracy: 0.8566 - recall: 0.8302 - precision: 0.8842 - AUROC: 0.9893 - AUPRC: 0.9374 - val_loss: 0.4019 - val_accuracy: 0.8575 - val_recall: 0.8404 - val_precision: 0.8784 - val_AUROC: 0.9874 - val_AUPRC: 0.9344
Epoch 21/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.3898 - accuracy: 0.8558 - recall: 0.8308 - precision: 0.8823 - AUROC: 0.9890 - AUPRC: 0.9366
Epoch 21: val_loss improved from 0.39420 to 0.38808, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3897 - accuracy: 0.8558 - recall: 0.8308 - precision: 0.8824 - AUROC: 0.9890 - AUPRC: 0.9367 - val_loss: 0.3881 - val_accuracy: 0.8641 - val_recall: 0.8408 - val_precision: 0.8853 - val_AUROC: 0.9886 - val_AUPRC: 0.9379
Epoch 22/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3774 - accuracy: 0.8605 - recall: 0.8344 - precision: 0.8855 - AUROC: 0.9897 - AUPRC: 0.9399
Epoch 22: val_loss did not improve from 0.38808
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3774 - accuracy: 0.8605 - recall: 0.8345 - precision: 0.8855 - AUROC: 0.9896 - AUPRC: 0.9399 - val_loss: 0.4052 - val_accuracy: 0.8602 - val_recall: 0.8444 - val_precision: 0.8767 - val_AUROC: 0.9871 - val_AUPRC: 0.9350
Epoch 23/100
1800/1800 [==============================] - ETA: 0s - loss: 0.3706 - accuracy: 0.8635 - recall: 0.8385 - precision: 0.8890 - AUROC: 0.9900 - AUPRC: 0.9420
Epoch 23: val_loss did not improve from 0.38808
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3706 - accuracy: 0.8635 - recall: 0.8385 - precision: 0.8890 - AUROC: 0.9900 - AUPRC: 0.9420 - val_loss: 0.3886 - val_accuracy: 0.8645 - val_recall: 0.8468 - val_precision: 0.8827 - val_AUROC: 0.9884 - val_AUPRC: 0.9374
Epoch 24/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3709 - accuracy: 0.8625 - recall: 0.8379 - precision: 0.8865 - AUROC: 0.9900 - AUPRC: 0.9416
Epoch 24: val_loss did not improve from 0.38808
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3709 - accuracy: 0.8625 - recall: 0.8379 - precision: 0.8865 - AUROC: 0.9900 - AUPRC: 0.9416 - val_loss: 0.3901 - val_accuracy: 0.8644 - val_recall: 0.8431 - val_precision: 0.8830 - val_AUROC: 0.9881 - val_AUPRC: 0.9365
Epoch 25/100
1796/1800 [============================>.] - ETA: 0s - loss: 0.3634 - accuracy: 0.8660 - recall: 0.8422 - precision: 0.8903 - AUROC: 0.9903 - AUPRC: 0.9437
Epoch 25: val_loss improved from 0.38808 to 0.38364, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3634 - accuracy: 0.8660 - recall: 0.8422 - precision: 0.8903 - AUROC: 0.9903 - AUPRC: 0.9437 - val_loss: 0.3836 - val_accuracy: 0.8675 - val_recall: 0.8473 - val_precision: 0.8884 - val_AUROC: 0.9882 - val_AUPRC: 0.9388
Epoch 26/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.3544 - accuracy: 0.8686 - recall: 0.8448 - precision: 0.8904 - AUROC: 0.9908 - AUPRC: 0.9458
Epoch 26: val_loss improved from 0.38364 to 0.38140, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3544 - accuracy: 0.8686 - recall: 0.8448 - precision: 0.8904 - AUROC: 0.9908 - AUPRC: 0.9458 - val_loss: 0.3814 - val_accuracy: 0.8675 - val_recall: 0.8495 - val_precision: 0.8892 - val_AUROC: 0.9886 - val_AUPRC: 0.9399
Epoch 27/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3528 - accuracy: 0.8678 - recall: 0.8462 - precision: 0.8902 - AUROC: 0.9909 - AUPRC: 0.9464
Epoch 27: val_loss improved from 0.38140 to 0.37158, saving model to training_vgg_gray\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3528 - accuracy: 0.8678 - recall: 0.8461 - precision: 0.8902 - AUROC: 0.9909 - AUPRC: 0.9464 - val_loss: 0.3716 - val_accuracy: 0.8745 - val_recall: 0.8521 - val_precision: 0.8945 - val_AUROC: 0.9892 - val_AUPRC: 0.9422
Epoch 28/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.3500 - accuracy: 0.8702 - recall: 0.8472 - precision: 0.8926 - AUROC: 0.9909 - AUPRC: 0.9470
Epoch 28: val_loss did not improve from 0.37158
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3500 - accuracy: 0.8702 - recall: 0.8472 - precision: 0.8926 - AUROC: 0.9909 - AUPRC: 0.9470 - val_loss: 0.3835 - val_accuracy: 0.8686 - val_recall: 0.8516 - val_precision: 0.8889 - val_AUROC: 0.9883 - val_AUPRC: 0.9397
Epoch 29/100
1796/1800 [============================>.] - ETA: 0s - loss: 0.3456 - accuracy: 0.8712 - recall: 0.8489 - precision: 0.8924 - AUROC: 0.9911 - AUPRC: 0.9482
Epoch 29: val_loss did not improve from 0.37158
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3458 - accuracy: 0.8711 - recall: 0.8488 - precision: 0.8923 - AUROC: 0.9911 - AUPRC: 0.9482 - val_loss: 0.3834 - val_accuracy: 0.8687 - val_recall: 0.8519 - val_precision: 0.8865 - val_AUROC: 0.9881 - val_AUPRC: 0.9403
Epoch 30/100
1800/1800 [==============================] - ETA: 0s - loss: 0.3431 - accuracy: 0.8730 - recall: 0.8517 - precision: 0.8942 - AUROC: 0.9911 - AUPRC: 0.9487
Epoch 30: val_loss did not improve from 0.37158
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3431 - accuracy: 0.8730 - recall: 0.8517 - precision: 0.8942 - AUROC: 0.9911 - AUPRC: 0.9487 - val_loss: 0.3840 - val_accuracy: 0.8706 - val_recall: 0.8577 - val_precision: 0.8867 - val_AUROC: 0.9879 - val_AUPRC: 0.9399
Epoch 31/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.3372 - accuracy: 0.8749 - recall: 0.8537 - precision: 0.8950 - AUROC: 0.9915 - AUPRC: 0.9503
Epoch 31: val_loss did not improve from 0.37158
1800/1800 [==============================] - 27s 15ms/step - loss: 0.3370 - accuracy: 0.8750 - recall: 0.8538 - precision: 0.8950 - AUROC: 0.9915 - AUPRC: 0.9503 - val_loss: 0.3810 - val_accuracy: 0.8704 - val_recall: 0.8531 - val_precision: 0.8877 - val_AUROC: 0.9882 - val_AUPRC: 0.9409
338/338 [==============================] - 6s 14ms/step - loss: 0.3809 - accuracy: 0.8704 - recall: 0.8531 - precision: 0.8877 - AUROC: 0.9882 - AUPRC: 0.9409
1it [15:33, 933.03s/it]
In [44]:
print(f"VGG16 Soccer CNN Gray Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate:
Accuracy : test - 0.8703703880310059
AUROC : test - 0.9881564378738403
AUPRC : test - 0.940942108631134
--------------------------------------------------------------------------------
In [45]:
# CNN VGG16 - Gray images - Metrics
CNN_metrics
Out[45]:
[{'test_evaluation': {'loss': 0.3809435963630676,
   'accuracy': 0.8703703880310059,
   'recall': 0.8530555367469788,
   'precision': 0.8876577615737915,
   'AUROC': 0.9881564378738403,
   'AUPRC': 0.940942108631134}}]
In [75]:
# Save results (save dictionary to vgg_gray_metrics.pkl file)
with open('results/vgg_gray_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
In [46]:
# Save Gray VGG16 Soccer CNN
vgg16_soccer_gray = CNN_models[0]
vgg16_soccer_gray.save_weights('modelWeights/vgg16_soccer_gray_model.h5')
In [39]:
checkpoint_path = "training_vgg_rgb/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)

# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                                 save_weights_only=True,
                                                 save_best_only=True,
                                                 verbose=1)
interrupt = False
In [ ]:
# Flag to load weights if training got interrupted
interrupt = True
In [40]:
print("---- VGG16 Soccer CNN RGB ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(rgb_input, labels_soccer))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([rgb_input[x]/255 for x in train_indices]), np.array([rgb_input[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer.iloc[train_indices], labels_soccer.iloc[test_indices]

    # Get model
    model = vgg16_soccer_model
    
    # Load weights if training got interrupted
    if interrupt:
        model.load_weights(checkpoint_path)

    CNN_holdout_metrics, CNN_holdout_history = train_model_cp(
        model,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size,
        cp_callback
    )

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(model)
---- VGG16 Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Epoch 1/100
1800/1800 [==============================] - ETA: 0s - loss: 0.8185 - accuracy: 0.7068 - recall: 0.5898 - precision: 0.8062 - AUROC: 0.9581 - AUPRC: 0.7899
Epoch 1: val_loss improved from inf to 0.56561, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 39s 18ms/step - loss: 0.8185 - accuracy: 0.7068 - recall: 0.5898 - precision: 0.8062 - AUROC: 0.9581 - AUPRC: 0.7899 - val_loss: 0.5656 - val_accuracy: 0.8043 - val_recall: 0.7478 - val_precision: 0.8513 - val_AUROC: 0.9789 - val_AUPRC: 0.8845
Epoch 2/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.5878 - accuracy: 0.7893 - recall: 0.7309 - precision: 0.8428 - AUROC: 0.9772 - AUPRC: 0.8750
Epoch 2: val_loss improved from 0.56561 to 0.49451, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5876 - accuracy: 0.7894 - recall: 0.7310 - precision: 0.8429 - AUROC: 0.9772 - AUPRC: 0.8750 - val_loss: 0.4945 - val_accuracy: 0.8237 - val_recall: 0.7775 - val_precision: 0.8638 - val_AUROC: 0.9837 - val_AUPRC: 0.9077
Epoch 3/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.5184 - accuracy: 0.8132 - recall: 0.7677 - precision: 0.8546 - AUROC: 0.9818 - AUPRC: 0.8981
Epoch 3: val_loss improved from 0.49451 to 0.45372, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.5183 - accuracy: 0.8131 - recall: 0.7676 - precision: 0.8546 - AUROC: 0.9818 - AUPRC: 0.8981 - val_loss: 0.4537 - val_accuracy: 0.8427 - val_recall: 0.8031 - val_precision: 0.8800 - val_AUROC: 0.9861 - val_AUPRC: 0.9212
Epoch 4/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4797 - accuracy: 0.8278 - recall: 0.7871 - precision: 0.8645 - AUROC: 0.9841 - AUPRC: 0.9107
Epoch 4: val_loss improved from 0.45372 to 0.42020, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4797 - accuracy: 0.8278 - recall: 0.7871 - precision: 0.8645 - AUROC: 0.9841 - AUPRC: 0.9107 - val_loss: 0.4202 - val_accuracy: 0.8569 - val_recall: 0.8223 - val_precision: 0.8848 - val_AUROC: 0.9874 - val_AUPRC: 0.9295
Epoch 5/100
1800/1800 [==============================] - ETA: 0s - loss: 0.4474 - accuracy: 0.8407 - recall: 0.8044 - precision: 0.8733 - AUROC: 0.9859 - AUPRC: 0.9204
Epoch 5: val_loss improved from 0.42020 to 0.41564, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4474 - accuracy: 0.8407 - recall: 0.8044 - precision: 0.8733 - AUROC: 0.9859 - AUPRC: 0.9204 - val_loss: 0.4156 - val_accuracy: 0.8539 - val_recall: 0.8215 - val_precision: 0.8832 - val_AUROC: 0.9876 - val_AUPRC: 0.9310
Epoch 6/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.4222 - accuracy: 0.8481 - recall: 0.8178 - precision: 0.8769 - AUROC: 0.9872 - AUPRC: 0.9279
Epoch 6: val_loss improved from 0.41564 to 0.39419, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4220 - accuracy: 0.8483 - recall: 0.8179 - precision: 0.8770 - AUROC: 0.9873 - AUPRC: 0.9280 - val_loss: 0.3942 - val_accuracy: 0.8612 - val_recall: 0.8350 - val_precision: 0.8862 - val_AUROC: 0.9885 - val_AUPRC: 0.9358
Epoch 7/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.4052 - accuracy: 0.8546 - recall: 0.8247 - precision: 0.8826 - AUROC: 0.9882 - AUPRC: 0.9331
Epoch 7: val_loss improved from 0.39419 to 0.37756, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.4053 - accuracy: 0.8546 - recall: 0.8248 - precision: 0.8826 - AUROC: 0.9882 - AUPRC: 0.9331 - val_loss: 0.3776 - val_accuracy: 0.8670 - val_recall: 0.8440 - val_precision: 0.8894 - val_AUROC: 0.9889 - val_AUPRC: 0.9405
Epoch 8/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.3840 - accuracy: 0.8632 - recall: 0.8360 - precision: 0.8884 - AUROC: 0.9893 - AUPRC: 0.9385
Epoch 8: val_loss did not improve from 0.37756
1800/1800 [==============================] - 28s 16ms/step - loss: 0.3840 - accuracy: 0.8632 - recall: 0.8360 - precision: 0.8884 - AUROC: 0.9893 - AUPRC: 0.9385 - val_loss: 0.3780 - val_accuracy: 0.8709 - val_recall: 0.8540 - val_precision: 0.8867 - val_AUROC: 0.9887 - val_AUPRC: 0.9414
Epoch 9/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3720 - accuracy: 0.8665 - recall: 0.8402 - precision: 0.8911 - AUROC: 0.9899 - AUPRC: 0.9418
Epoch 9: val_loss improved from 0.37756 to 0.36596, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3719 - accuracy: 0.8665 - recall: 0.8402 - precision: 0.8911 - AUROC: 0.9900 - AUPRC: 0.9418 - val_loss: 0.3660 - val_accuracy: 0.8723 - val_recall: 0.8544 - val_precision: 0.8916 - val_AUROC: 0.9890 - val_AUPRC: 0.9442
Epoch 10/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.3568 - accuracy: 0.8708 - recall: 0.8474 - precision: 0.8949 - AUROC: 0.9906 - AUPRC: 0.9460
Epoch 10: val_loss improved from 0.36596 to 0.36201, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3568 - accuracy: 0.8708 - recall: 0.8473 - precision: 0.8948 - AUROC: 0.9906 - AUPRC: 0.9460 - val_loss: 0.3620 - val_accuracy: 0.8752 - val_recall: 0.8544 - val_precision: 0.8937 - val_AUROC: 0.9896 - val_AUPRC: 0.9453
Epoch 11/100
1800/1800 [==============================] - ETA: 0s - loss: 0.3467 - accuracy: 0.8724 - recall: 0.8498 - precision: 0.8948 - AUROC: 0.9911 - AUPRC: 0.9486
Epoch 11: val_loss improved from 0.36201 to 0.34957, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3467 - accuracy: 0.8724 - recall: 0.8498 - precision: 0.8948 - AUROC: 0.9911 - AUPRC: 0.9486 - val_loss: 0.3496 - val_accuracy: 0.8825 - val_recall: 0.8661 - val_precision: 0.9003 - val_AUROC: 0.9898 - val_AUPRC: 0.9486
Epoch 12/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3339 - accuracy: 0.8782 - recall: 0.8575 - precision: 0.8991 - AUROC: 0.9916 - AUPRC: 0.9520
Epoch 12: val_loss improved from 0.34957 to 0.33201, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3342 - accuracy: 0.8781 - recall: 0.8574 - precision: 0.8990 - AUROC: 0.9916 - AUPRC: 0.9519 - val_loss: 0.3320 - val_accuracy: 0.8885 - val_recall: 0.8724 - val_precision: 0.9050 - val_AUROC: 0.9907 - val_AUPRC: 0.9530
Epoch 13/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.3248 - accuracy: 0.8831 - recall: 0.8628 - precision: 0.9026 - AUROC: 0.9919 - AUPRC: 0.9540
Epoch 13: val_loss did not improve from 0.33201
1800/1800 [==============================] - 28s 15ms/step - loss: 0.3248 - accuracy: 0.8831 - recall: 0.8628 - precision: 0.9026 - AUROC: 0.9919 - AUPRC: 0.9540 - val_loss: 0.3371 - val_accuracy: 0.8865 - val_recall: 0.8724 - val_precision: 0.9038 - val_AUROC: 0.9902 - val_AUPRC: 0.9515
Epoch 14/100
1800/1800 [==============================] - ETA: 0s - loss: 0.3153 - accuracy: 0.8874 - recall: 0.8681 - precision: 0.9053 - AUROC: 0.9924 - AUPRC: 0.9563
Epoch 14: val_loss improved from 0.33201 to 0.33186, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3153 - accuracy: 0.8874 - recall: 0.8681 - precision: 0.9053 - AUROC: 0.9924 - AUPRC: 0.9563 - val_loss: 0.3319 - val_accuracy: 0.8884 - val_recall: 0.8741 - val_precision: 0.9037 - val_AUROC: 0.9906 - val_AUPRC: 0.9536
Epoch 15/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.3100 - accuracy: 0.8870 - recall: 0.8692 - precision: 0.9053 - AUROC: 0.9926 - AUPRC: 0.9575
Epoch 15: val_loss improved from 0.33186 to 0.32336, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.3100 - accuracy: 0.8870 - recall: 0.8693 - precision: 0.9053 - AUROC: 0.9926 - AUPRC: 0.9575 - val_loss: 0.3234 - val_accuracy: 0.8921 - val_recall: 0.8755 - val_precision: 0.9081 - val_AUROC: 0.9911 - val_AUPRC: 0.9553
Epoch 16/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.2988 - accuracy: 0.8904 - recall: 0.8735 - precision: 0.9082 - AUROC: 0.9931 - AUPRC: 0.9601
Epoch 16: val_loss did not improve from 0.32336
1800/1800 [==============================] - 28s 16ms/step - loss: 0.2989 - accuracy: 0.8904 - recall: 0.8735 - precision: 0.9082 - AUROC: 0.9931 - AUPRC: 0.9601 - val_loss: 0.3297 - val_accuracy: 0.8908 - val_recall: 0.8779 - val_precision: 0.9071 - val_AUROC: 0.9905 - val_AUPRC: 0.9538
Epoch 17/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.2987 - accuracy: 0.8921 - recall: 0.8761 - precision: 0.9098 - AUROC: 0.9930 - AUPRC: 0.9602
Epoch 17: val_loss did not improve from 0.32336
1800/1800 [==============================] - 28s 16ms/step - loss: 0.2987 - accuracy: 0.8922 - recall: 0.8762 - precision: 0.9099 - AUROC: 0.9930 - AUPRC: 0.9602 - val_loss: 0.3260 - val_accuracy: 0.8898 - val_recall: 0.8732 - val_precision: 0.9079 - val_AUROC: 0.9908 - val_AUPRC: 0.9548
Epoch 18/100
1796/1800 [============================>.] - ETA: 0s - loss: 0.2893 - accuracy: 0.8942 - recall: 0.8784 - precision: 0.9116 - AUROC: 0.9935 - AUPRC: 0.9623
Epoch 18: val_loss did not improve from 0.32336
1800/1800 [==============================] - 28s 15ms/step - loss: 0.2894 - accuracy: 0.8942 - recall: 0.8784 - precision: 0.9116 - AUROC: 0.9935 - AUPRC: 0.9622 - val_loss: 0.3235 - val_accuracy: 0.8931 - val_recall: 0.8778 - val_precision: 0.9084 - val_AUROC: 0.9909 - val_AUPRC: 0.9552
Epoch 19/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.2791 - accuracy: 0.8986 - recall: 0.8828 - precision: 0.9155 - AUROC: 0.9938 - AUPRC: 0.9644
Epoch 19: val_loss improved from 0.32336 to 0.31182, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.2790 - accuracy: 0.8986 - recall: 0.8828 - precision: 0.9155 - AUROC: 0.9938 - AUPRC: 0.9644 - val_loss: 0.3118 - val_accuracy: 0.8946 - val_recall: 0.8830 - val_precision: 0.9072 - val_AUROC: 0.9915 - val_AUPRC: 0.9583
Epoch 20/100
1798/1800 [============================>.] - ETA: 0s - loss: 0.2747 - accuracy: 0.8985 - recall: 0.8835 - precision: 0.9137 - AUROC: 0.9940 - AUPRC: 0.9654
Epoch 20: val_loss improved from 0.31182 to 0.31141, saving model to training_vgg_rgb\cp.ckpt
1800/1800 [==============================] - 29s 16ms/step - loss: 0.2749 - accuracy: 0.8984 - recall: 0.8834 - precision: 0.9136 - AUROC: 0.9940 - AUPRC: 0.9654 - val_loss: 0.3114 - val_accuracy: 0.8994 - val_recall: 0.8875 - val_precision: 0.9109 - val_AUROC: 0.9913 - val_AUPRC: 0.9587
Epoch 21/100
1797/1800 [============================>.] - ETA: 0s - loss: 0.2683 - accuracy: 0.9025 - recall: 0.8876 - precision: 0.9176 - AUROC: 0.9942 - AUPRC: 0.9667
Epoch 21: val_loss did not improve from 0.31141
1800/1800 [==============================] - 28s 15ms/step - loss: 0.2684 - accuracy: 0.9024 - recall: 0.8875 - precision: 0.9175 - AUROC: 0.9942 - AUPRC: 0.9667 - val_loss: 0.3253 - val_accuracy: 0.8940 - val_recall: 0.8844 - val_precision: 0.9060 - val_AUROC: 0.9905 - val_AUPRC: 0.9561
Epoch 22/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.2645 - accuracy: 0.9023 - recall: 0.8883 - precision: 0.9169 - AUROC: 0.9945 - AUPRC: 0.9675
Epoch 22: val_loss did not improve from 0.31141
1800/1800 [==============================] - 28s 15ms/step - loss: 0.2644 - accuracy: 0.9023 - recall: 0.8883 - precision: 0.9169 - AUROC: 0.9945 - AUPRC: 0.9675 - val_loss: 0.3222 - val_accuracy: 0.8943 - val_recall: 0.8831 - val_precision: 0.9070 - val_AUROC: 0.9909 - val_AUPRC: 0.9568
Epoch 23/100
1796/1800 [============================>.] - ETA: 0s - loss: 0.2631 - accuracy: 0.9034 - recall: 0.8899 - precision: 0.9177 - AUROC: 0.9944 - AUPRC: 0.9676
Epoch 23: val_loss did not improve from 0.31141
1800/1800 [==============================] - 28s 15ms/step - loss: 0.2631 - accuracy: 0.9034 - recall: 0.8899 - precision: 0.9177 - AUROC: 0.9944 - AUPRC: 0.9676 - val_loss: 0.3253 - val_accuracy: 0.8947 - val_recall: 0.8878 - val_precision: 0.9069 - val_AUROC: 0.9906 - val_AUPRC: 0.9564
Epoch 24/100
1799/1800 [============================>.] - ETA: 0s - loss: 0.2589 - accuracy: 0.9043 - recall: 0.8915 - precision: 0.9175 - AUROC: 0.9946 - AUPRC: 0.9687
Epoch 24: val_loss did not improve from 0.31141
1800/1800 [==============================] - 28s 15ms/step - loss: 0.2588 - accuracy: 0.9043 - recall: 0.8916 - precision: 0.9175 - AUROC: 0.9946 - AUPRC: 0.9687 - val_loss: 0.3252 - val_accuracy: 0.9011 - val_recall: 0.8919 - val_precision: 0.9120 - val_AUROC: 0.9900 - val_AUPRC: 0.9564
338/338 [==============================] - 6s 14ms/step - loss: 0.3252 - accuracy: 0.9011 - recall: 0.8919 - precision: 0.9120 - AUROC: 0.9900 - AUPRC: 0.9564
1it [12:32, 752.18s/it]
In [42]:
print(f"VGG16 Soccer CNN RGB Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 1-holdouts estimate:
Accuracy : test - 0.9011111259460449
AUROC : test - 0.990019679069519
AUPRC : test - 0.9564046263694763
--------------------------------------------------------------------------------
In [43]:
# CNN VGG16 - RGB Images - Metrics
CNN_metrics
Out[43]:
[{'test_evaluation': {'loss': 0.3251974582672119,
   'accuracy': 0.9011111259460449,
   'recall': 0.8919444680213928,
   'precision': 0.9120431542396545,
   'AUROC': 0.990019679069519,
   'AUPRC': 0.9564046263694763}}]
In [77]:
# Save results (save dictionary to vgg_rgb_metrics.pkl file)
with open('results/vgg_rgb_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file
In [44]:
# Save RGB VGG16 Soccer CNN
vgg16_soccer_rgb = CNN_models[0]
vgg16_soccer_rgb.save_weights('modelWeights/vgg16_soccer_rgb_model.h5')

Evaluation of models on the "Real Soccer" dataset¶

Evaluation on images not from the football analysis dataset to evaluate the generalization capabilities of the models on different kinds of soccer images.

In [37]:
image_shape_gray = (soccer_images_dim,soccer_images_dim,1)
In [53]:
## Load Models
oneDim_models = []
threeDim_models = []

# Create models
soccer = build_soccer_CNN(image_shape_gray)
soccer_data_augmentation = build_soccer_CNN(image_shape_gray)
soccer_resNet_model = resNet_soccer_model
soccer_vgg_model = vgg16_soccer_model

# Restore the weights
soccer.load_weights('modelWeights/soccerCNN_Model.h5')
soccer._name = "Basic_CNN"

soccer_data_augmentation.load_weights('modelWeights/soccerModelDataAugmentation.h5')
soccer_data_augmentation._name = "CNN_Data_Augmentation"

soccer_resNet_model.load_weights('modelWeights/resNet_soccer_gray_model.h5')
soccer_resNet_model._name = "ResNet_Soccer_Gray"

soccer_vgg_model.load_weights('modelWeights/vgg16_soccer_gray_model.h5')
soccer_vgg_model._name = "VGG16_Soccer_Gray"

oneDim_models.append(soccer)
oneDim_models.append(soccer_data_augmentation)
threeDim_models.append(soccer_resNet_model)
threeDim_models.append(soccer_vgg_model)
Model: "Soccer_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_16 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_16 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_17 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_17 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_18 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_18 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 conv2d_19 (Conv2D)          (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_19 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_5 (Flatten)         (None, 6400)              0         
                                                                 
 dense_17 (Dense)            (None, 128)               819328    
                                                                 
 dropout_11 (Dropout)        (None, 128)               0         
                                                                 
 dense_18 (Dense)            (None, 128)               16512     
                                                                 
 dropout_12 (Dropout)        (None, 128)               0         
                                                                 
 dense_19 (Dense)            (None, 9)                 1161      
                                                                 
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
Model: "Soccer_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_20 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_20 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_21 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_21 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_22 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_22 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 conv2d_23 (Conv2D)          (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_23 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_6 (Flatten)         (None, 6400)              0         
                                                                 
 dense_20 (Dense)            (None, 128)               819328    
                                                                 
 dropout_13 (Dropout)        (None, 128)               0         
                                                                 
 dense_21 (Dense)            (None, 128)               16512     
                                                                 
 dropout_14 (Dropout)        (None, 128)               0         
                                                                 
 dense_22 (Dense)            (None, 9)                 1161      
                                                                 
=================================================================
Total params: 1,871,113
Trainable params: 1,871,113
Non-trainable params: 0
_________________________________________________________________
In [46]:
from sklearn.metrics import roc_curve, auc, roc_auc_score, precision_recall_curve, average_precision_score
from sklearn.preprocessing import label_binarize

def get_metrics(labels, predictions):
    """ Returns the AUROC and AUPRC metrics given the true labels and the model predictions.
    
    Parameters
    ------------------------
    labels: tuple
        The true labels (n_samples,).
    predictions: np.array
        The model predictions (n_samples,).
        
    Returns
    ------------------------
    accuracy: float
        The accuracy metric.
    auroc: float
        The AUROC metric.
    auprc: float
        The AUPRC metric."""
    
    # Change shape from (n_samples,) to (n_samples, n_classes)
    labels = label_binarize(labels, classes=np.unique(labels))
    predictions = label_binarize(predictions, classes=np.unique(predictions)) 
    
    # Accuracy
    accuracy = accuracy_score(labels, predictions)
    
    # AUROC
    auroc = roc_auc_score(labels, predictions)
        
    # AUPRC
    auprc = average_precision_score(labels, predictions)

    return accuracy, auroc, auprc
In [47]:
def one_hot_max(arr):
    """ Returns the index of maximum value in the array (prediction). """ 
    max_index = np.argmax(arr)
    
    one_hot_arr = np.zeros_like(arr)
    one_hot_arr[max_index] = 1
    
    return one_hot_arr
In [48]:
# Dataset preparation
real_data_soccer_all = []
for event in real_data_soccer:
    for element in real_data_soccer[event]:
        real_data_soccer_all.append(element)

print(f"Data shape: {np.array(real_data_soccer_all).shape}")
print(f"Labels shape: {real_labels_soccer.shape}")

# 3 Channel data for resNet and VGG16
threeChannels_real_input = np.array([cv2.cvtColor(x, cv2.COLOR_GRAY2RGB) for x in real_data_soccer_all])
threeChannels_real_input = np.array([x/255 for x in threeChannels_real_input])
Data shape: (90, 80, 80)
Labels shape: (90, 9)
In [49]:
real_data_soccer_all = np.array([np.expand_dims(x/255, axis=-1) for x in real_data_soccer_all])
In [44]:
for model in oneDim_models:
    real_data_soccer_predictions = get_predictions(model, real_data_soccer_all, 2)
          
    # Shape (n_samples,)
    labels = np.argmax(real_labels_soccer, axis=1)
    predictions = np.argmax(real_data_soccer_predictions, axis=1)

    acc, auroc, auprc = get_metrics(
        labels = labels,
        predictions = predictions
    )
    print(f"{model.name} metrics on real dataset:")
    print(f"Accuracy: {acc}")
    print("-"*30)
    print(f"AUROC: {auroc}")
    print("-"*30)
    print(f"AUPRC: {auprc}")
    print("-"*80)
    
    true_labels = labels
    predicted_labels = predictions

    confusion_matrix = tf.math.confusion_matrix(
        true_labels,
        predicted_labels,
        num_classes=9,
        weights=None,
        name=None
    )

    sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
    sns.set (rc = {'figure.figsize':(8,8)})
    plt.xlabel('Predicted Labels', fontsize=12)
    plt.ylabel('True Labels', fontsize=12)
    plt.yticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11")
    plt.xticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11" )
    plt.show()
    
for model in threeDim_models:
    real_data_soccer_predictions = get_predictions(model, threeChannels_real_input, 2)
          
    # Shape (n_samples,)
    labels = labels
    predictions = np.argmax(real_data_soccer_predictions, axis=1)

    acc, auroc, auprc = get_metrics(
        labels = labels,
        predictions = predictions
    )
    print(f"{model.name} metrics on real dataset:")
    print(f"Accuracy: {acc}")
    print("-"*30)
    print(f"AUROC: {auroc}")
    print("-"*30)
    print(f"AUPRC: {auprc}")
    print("-"*80)
    
    true_labels = labels
    predicted_labels = predictions

    confusion_matrix = tf.math.confusion_matrix(
        true_labels,
        predicted_labels,
        num_classes=9,
        weights=None,
        name=None
    )

    sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
    sns.set (rc = {'figure.figsize':(8,8)})
    plt.xlabel('Predicted Labels', fontsize=12)
    plt.ylabel('True Labels', fontsize=12)
    plt.yticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11")
    plt.xticks(np.arange(9)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute'), fontsize="11" )
    plt.show()
45/45 [==============================] - 0s 6ms/step
Basic_CNN metrics on real dataset:
Accuracy: 0.6888888888888889
------------------------------
AUROC: 0.825
------------------------------
AUPRC: 0.5385870302536969
--------------------------------------------------------------------------------
No description has been provided for this image
45/45 [==============================] - 0s 1ms/step
CNN_Data_Augmentation metrics on real dataset:
Accuracy: 0.7
------------------------------
AUROC: 0.83125
------------------------------
AUPRC: 0.572998366013072
--------------------------------------------------------------------------------
No description has been provided for this image
45/45 [==============================] - 0s 8ms/step
ResNet_Soccer_Gray metrics on real dataset:
Accuracy: 0.5444444444444444
------------------------------
AUROC: 0.74375
------------------------------
AUPRC: 0.39875423958757295
--------------------------------------------------------------------------------
No description has been provided for this image
45/45 [==============================] - 0s 3ms/step
VGG16_Soccer_Gray metrics on real dataset:
Accuracy: 0.7
------------------------------
AUROC: 0.8312500000000002
------------------------------
AUPRC: 0.5530253696920363
--------------------------------------------------------------------------------
No description has been provided for this image

Card classification (Red / Yellow)¶

Images previously classified as Cards are to be classified in Red vs Yellow.
The cards classifier is a CNN taking in input RGB images to exploit the required color information to perform the task.

In [103]:
folderCards = r'D:\MachineLearning\Datasets\Soccer\TrainTestCards'

data_cards, labels_cards = loadData(folderCards, True)
In [104]:
print("[n.images, (width, height, depth)]")
for folder in (data_cards):
    print(f"--- {folder} ---")
    print(f"shape: {len(data_cards[folder])} {data_cards[folder][1].shape}")

card_sample = data_cards['Red-Cards'][22]

print("-"*30)

print(f"label {labels_cards[22]}")

plt.subplot(1,2,1),plt.imshow(card_sample, cmap='gray', vmin=0, vmax=255)
[n.images, (width, height, depth)]
--- Red-Cards ---
shape: 6000 (80, 80, 3)
--- Yellow-Cards ---
shape: 6000 (80, 80, 3)
------------------------------
label Red-Cards
Out[104]:
(<Axes: >, <matplotlib.image.AxesImage at 0x161fa5e7cd0>)
No description has been provided for this image
In [105]:
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()

labels_cards = pd.DataFrame(le.fit_transform(labels_cards))

print(labels_cards.sample(3))
      0
8539  1
9951  1
4806  0
In [106]:
# data_cards is a dict containing ['events'...] so group together classes to evaluate
data_cards_all = []
for event in data_cards:
    for element in data_cards[event]:
        data_cards_all.append(element)

print(f"Data shape: {np.array(data_cards_all).shape}")
print(f"Labels shape: {labels_cards.shape}")
Data shape: (12000, 80, 80, 3)
Labels shape: (12000, 1)
In [107]:
def build_cards_CNN(
    input_shape: tuple
) -> tf.keras.Model:
    """ Returns the Cards CNN model.

    Parameters
    ------------------------
    input_shape: tuple
        The shape of the input.

    Returns
    ------------------------
    model: model
        The Cards model."""

    CNN = Sequential(name="cards_CNN")
    CNN.add(layers.Input(input_shape))
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 256,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),

    CNN.add(layers.Flatten())
    
    for i in range(2):
        CNN.add(layers.Dense(units = 128, activation='relu'))
        CNN.add(layers.Dropout(0.5))
        
    CNN.add(layers.Dense(2, activation='sigmoid'))
    
    CNN.compile(
        optimizer='adam',
        loss='binary_crossentropy',
        metrics=get_standard_binary_metrics()
    )
    
    CNN.summary()
    
    return CNN

Main training loop - Cards CNN¶

  • Normalize RGB images' values in 0-1 range.
  • Build and train the model over the training set and test it over the test set for the different holdouts.
In [109]:
number_of_splits = 2

holdouts_generator = StratifiedShuffleSplit(
    n_splits = number_of_splits,
    test_size = 0.2,
    random_state = 42
)
In [110]:
epochs = 100
batch_size = 32
In [111]:
print("---- CARDS CNN ----")

CNN_cards_metrics = []
CNN_cards_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_cards_all, labels_cards))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([data_cards_all[x]/255 for x in train_indices]), np.array([data_cards_all[x]/255 for x in test_indices])
    y_train, y_test = labels_cards.iloc[train_indices], labels_cards.iloc[test_indices]
    
    # One hot encoding
    y_train = one_hot_encoding(y_train, 2)
    y_test = one_hot_encoding(y_test, 2)

    # Build CNN
    CNN = build_cards_CNN(x_train[0].shape)

    print("- Training model:\n")
    CNN_holdout_metrics, CNN_holdout_history = train_model(
        CNN,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size
    )
    
    predictions = get_predictions(CNN, x_test, batch_size=batch_size)
    CNN_holdout_predictions.append(predictions)
    CNN_holdout_test_labels.append(y_test)

    CNN_cards_metrics.append(CNN_holdout_metrics)
    CNN_cards_history.append(CNN_holdout_history)
    
    CNN_models.append(CNN)
---- CARDS CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "cards_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_24 (Conv2D)          (None, 80, 80, 128)       3584      
                                                                 
 max_pooling2d_24 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_25 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_25 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_26 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_26 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 flatten_7 (Flatten)         (None, 25600)             0         
                                                                 
 dense_23 (Dense)            (None, 128)               3276928   
                                                                 
 dropout_15 (Dropout)        (None, 128)               0         
                                                                 
 dense_24 (Dense)            (None, 128)               16512     
                                                                 
 dropout_16 (Dropout)        (None, 128)               0         
                                                                 
 dense_25 (Dense)            (None, 2)                 258       
                                                                 
=================================================================
Total params: 3,740,034
Trainable params: 3,740,034
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
300/300 [==============================] - 8s 19ms/step - loss: 0.5340 - accuracy: 0.7060 - recall: 0.7081 - precision: 0.7045 - AUROC: 0.8004 - AUPRC: 0.8048 - f1_score: 0.7063 - balanced_accuracy: 0.7055 - specificity: 0.7029 - miss_rate: 0.2919 - fall_out: 0.2971 - mcc: 0.4110 - val_loss: 0.2701 - val_accuracy: 0.8917 - val_recall: 0.8917 - val_precision: 0.8920 - val_AUROC: 0.9557 - val_AUPRC: 0.9553 - val_f1_score: 0.8919 - val_balanced_accuracy: 0.8919 - val_specificity: 0.8921 - val_miss_rate: 0.1083 - val_fall_out: 0.1079 - val_mcc: 0.7838
Epoch 2/100
300/300 [==============================] - 5s 17ms/step - loss: 0.2000 - accuracy: 0.9285 - recall: 0.9275 - precision: 0.9288 - AUROC: 0.9760 - AUPRC: 0.9750 - f1_score: 0.9281 - balanced_accuracy: 0.9282 - specificity: 0.9289 - miss_rate: 0.0725 - fall_out: 0.0711 - mcc: 0.8564 - val_loss: 0.1520 - val_accuracy: 0.9367 - val_recall: 0.9379 - val_precision: 0.9360 - val_AUROC: 0.9858 - val_AUPRC: 0.9860 - val_f1_score: 0.9369 - val_balanced_accuracy: 0.9369 - val_specificity: 0.9358 - val_miss_rate: 0.0621 - val_fall_out: 0.0642 - val_mcc: 0.8738
Epoch 3/100
300/300 [==============================] - 5s 17ms/step - loss: 0.1253 - accuracy: 0.9543 - recall: 0.9542 - precision: 0.9544 - AUROC: 0.9899 - AUPRC: 0.9894 - f1_score: 0.9543 - balanced_accuracy: 0.9543 - specificity: 0.9544 - miss_rate: 0.0458 - fall_out: 0.0456 - mcc: 0.9085 - val_loss: 0.2360 - val_accuracy: 0.9267 - val_recall: 0.9279 - val_precision: 0.9271 - val_AUROC: 0.9762 - val_AUPRC: 0.9728 - val_f1_score: 0.9275 - val_balanced_accuracy: 0.9275 - val_specificity: 0.9271 - val_miss_rate: 0.0721 - val_fall_out: 0.0729 - val_mcc: 0.8550
Epoch 4/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0964 - accuracy: 0.9658 - recall: 0.9656 - precision: 0.9654 - AUROC: 0.9939 - AUPRC: 0.9937 - f1_score: 0.9655 - balanced_accuracy: 0.9655 - specificity: 0.9654 - miss_rate: 0.0344 - fall_out: 0.0346 - mcc: 0.9310 - val_loss: 0.1289 - val_accuracy: 0.9529 - val_recall: 0.9529 - val_precision: 0.9533 - val_AUROC: 0.9901 - val_AUPRC: 0.9898 - val_f1_score: 0.9531 - val_balanced_accuracy: 0.9531 - val_specificity: 0.9533 - val_miss_rate: 0.0471 - val_fall_out: 0.0467 - val_mcc: 0.9063
Epoch 5/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0696 - accuracy: 0.9761 - recall: 0.9757 - precision: 0.9763 - AUROC: 0.9966 - AUPRC: 0.9964 - f1_score: 0.9760 - balanced_accuracy: 0.9760 - specificity: 0.9764 - miss_rate: 0.0243 - fall_out: 0.0236 - mcc: 0.9521 - val_loss: 0.1453 - val_accuracy: 0.9467 - val_recall: 0.9463 - val_precision: 0.9463 - val_AUROC: 0.9877 - val_AUPRC: 0.9865 - val_f1_score: 0.9463 - val_balanced_accuracy: 0.9463 - val_specificity: 0.9463 - val_miss_rate: 0.0538 - val_fall_out: 0.0538 - val_mcc: 0.8925
Epoch 6/100
300/300 [==============================] - 5s 16ms/step - loss: 0.0610 - accuracy: 0.9772 - recall: 0.9770 - precision: 0.9773 - AUROC: 0.9974 - AUPRC: 0.9974 - f1_score: 0.9771 - balanced_accuracy: 0.9771 - specificity: 0.9773 - miss_rate: 0.0230 - fall_out: 0.0227 - mcc: 0.9543 - val_loss: 0.1364 - val_accuracy: 0.9571 - val_recall: 0.9579 - val_precision: 0.9571 - val_AUROC: 0.9906 - val_AUPRC: 0.9890 - val_f1_score: 0.9575 - val_balanced_accuracy: 0.9575 - val_specificity: 0.9571 - val_miss_rate: 0.0421 - val_fall_out: 0.0429 - val_mcc: 0.9150
300/300 [==============================] - 2s 6ms/step - loss: 0.0381 - accuracy: 0.9864 - recall: 0.9860 - precision: 0.9861 - AUROC: 0.9987 - AUPRC: 0.9985 - f1_score: 0.9861 - balanced_accuracy: 0.9861 - specificity: 0.9861 - miss_rate: 0.0140 - fall_out: 0.0139 - mcc: 0.9722
75/75 [==============================] - 0s 6ms/step - loss: 0.1364 - accuracy: 0.9571 - recall: 0.9579 - precision: 0.9571 - AUROC: 0.9906 - AUPRC: 0.9890 - f1_score: 0.9575 - balanced_accuracy: 0.9575 - specificity: 0.9571 - miss_rate: 0.0421 - fall_out: 0.0429 - mcc: 0.9150
75/75 [==============================] - 0s 4ms/step
1it [00:38, 38.92s/it]
-- HOLDOUT 2
Model: "cards_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_27 (Conv2D)          (None, 80, 80, 128)       3584      
                                                                 
 max_pooling2d_27 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_28 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_28 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_29 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_29 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 flatten_8 (Flatten)         (None, 25600)             0         
                                                                 
 dense_26 (Dense)            (None, 128)               3276928   
                                                                 
 dropout_17 (Dropout)        (None, 128)               0         
                                                                 
 dense_27 (Dense)            (None, 128)               16512     
                                                                 
 dropout_18 (Dropout)        (None, 128)               0         
                                                                 
 dense_28 (Dense)            (None, 2)                 258       
                                                                 
=================================================================
Total params: 3,740,034
Trainable params: 3,740,034
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
300/300 [==============================] - 8s 20ms/step - loss: 0.4929 - accuracy: 0.7400 - recall: 0.7268 - precision: 0.7414 - AUROC: 0.8344 - AUPRC: 0.8392 - f1_score: 0.7340 - balanced_accuracy: 0.7367 - specificity: 0.7466 - miss_rate: 0.2732 - fall_out: 0.2534 - mcc: 0.4734 - val_loss: 0.2235 - val_accuracy: 0.9158 - val_recall: 0.9146 - val_precision: 0.9161 - val_AUROC: 0.9698 - val_AUPRC: 0.9694 - val_f1_score: 0.9153 - val_balanced_accuracy: 0.9154 - val_specificity: 0.9162 - val_miss_rate: 0.0854 - val_fall_out: 0.0838 - val_mcc: 0.8308
Epoch 2/100
300/300 [==============================] - 5s 17ms/step - loss: 0.1735 - accuracy: 0.9357 - recall: 0.9370 - precision: 0.9358 - AUROC: 0.9815 - AUPRC: 0.9809 - f1_score: 0.9364 - balanced_accuracy: 0.9364 - specificity: 0.9357 - miss_rate: 0.0630 - fall_out: 0.0643 - mcc: 0.8727 - val_loss: 0.1530 - val_accuracy: 0.9471 - val_recall: 0.9479 - val_precision: 0.9471 - val_AUROC: 0.9854 - val_AUPRC: 0.9845 - val_f1_score: 0.9475 - val_balanced_accuracy: 0.9475 - val_specificity: 0.9471 - val_miss_rate: 0.0521 - val_fall_out: 0.0529 - val_mcc: 0.8950
Epoch 3/100
300/300 [==============================] - 5s 18ms/step - loss: 0.1166 - accuracy: 0.9579 - recall: 0.9577 - precision: 0.9576 - AUROC: 0.9913 - AUPRC: 0.9911 - f1_score: 0.9577 - balanced_accuracy: 0.9577 - specificity: 0.9576 - miss_rate: 0.0423 - fall_out: 0.0424 - mcc: 0.9153 - val_loss: 0.1239 - val_accuracy: 0.9592 - val_recall: 0.9588 - val_precision: 0.9595 - val_AUROC: 0.9900 - val_AUPRC: 0.9889 - val_f1_score: 0.9591 - val_balanced_accuracy: 0.9592 - val_specificity: 0.9596 - val_miss_rate: 0.0413 - val_fall_out: 0.0404 - val_mcc: 0.9183
Epoch 4/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0861 - accuracy: 0.9702 - recall: 0.9696 - precision: 0.9697 - AUROC: 0.9950 - AUPRC: 0.9948 - f1_score: 0.9696 - balanced_accuracy: 0.9696 - specificity: 0.9697 - miss_rate: 0.0304 - fall_out: 0.0303 - mcc: 0.9393 - val_loss: 0.1796 - val_accuracy: 0.9442 - val_recall: 0.9446 - val_precision: 0.9434 - val_AUROC: 0.9842 - val_AUPRC: 0.9824 - val_f1_score: 0.9440 - val_balanced_accuracy: 0.9440 - val_specificity: 0.9433 - val_miss_rate: 0.0554 - val_fall_out: 0.0567 - val_mcc: 0.8879
Epoch 5/100
300/300 [==============================] - 5s 17ms/step - loss: 0.0736 - accuracy: 0.9709 - recall: 0.9712 - precision: 0.9710 - AUROC: 0.9963 - AUPRC: 0.9961 - f1_score: 0.9711 - balanced_accuracy: 0.9711 - specificity: 0.9710 - miss_rate: 0.0288 - fall_out: 0.0290 - mcc: 0.9423 - val_loss: 0.1295 - val_accuracy: 0.9546 - val_recall: 0.9550 - val_precision: 0.9538 - val_AUROC: 0.9900 - val_AUPRC: 0.9891 - val_f1_score: 0.9544 - val_balanced_accuracy: 0.9544 - val_specificity: 0.9538 - val_miss_rate: 0.0450 - val_fall_out: 0.0463 - val_mcc: 0.9088
300/300 [==============================] - 2s 6ms/step - loss: 0.0500 - accuracy: 0.9847 - recall: 0.9847 - precision: 0.9845 - AUROC: 0.9987 - AUPRC: 0.9987 - f1_score: 0.9846 - balanced_accuracy: 0.9846 - specificity: 0.9845 - miss_rate: 0.0153 - fall_out: 0.0155 - mcc: 0.9692
75/75 [==============================] - 0s 6ms/step - loss: 0.1295 - accuracy: 0.9546 - recall: 0.9550 - precision: 0.9538 - AUROC: 0.9900 - AUPRC: 0.9891 - f1_score: 0.9544 - balanced_accuracy: 0.9544 - specificity: 0.9538 - miss_rate: 0.0450 - fall_out: 0.0463 - mcc: 0.9088
75/75 [==============================] - 0s 4ms/step
2it [01:16, 38.09s/it]
In [112]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_cards_metrics, number_of_splits)

print(f"Red VS Yellow Card - CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']}  --  test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']}  --  test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']}  --  test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN - Train history:")
plot_train_history(CNN_cards_history)
print("-"*100)
Red VS Yellow Card - CNN Metrics - 2-holdouts estimate:
Accuracy : train - 0.9855208396911621  --  test - 0.9558333456516266
AUROC : train - 0.9987120628356934  --  test - 0.990309476852417
AUPRC : train - 0.9986125230789185  --  test - 0.9890577793121338
--------------------------------------------------------------------------------
CNN - Train history:
No description has been provided for this image
No description has been provided for this image
----------------------------------------------------------------------------------------------------
In [113]:
# Save Cards CNN
cards_model = CNN_models[0]
cards_model.save_weights('modelWeights/cards_model.h5')
In [114]:
# Cards CNN Metrics
CNN_cards_metrics[0]
Out[114]:
{'train_evaluation': {'loss': 0.03812405839562416,
  'accuracy': 0.9863541722297668,
  'recall': 0.9860416650772095,
  'precision': 0.9861443638801575,
  'AUROC': 0.9986873865127563,
  'AUPRC': 0.9985128045082092,
  'f1_score': 0.9860930442810059,
  'balanced_accuracy': 0.9860937595367432,
  'specificity': 0.9861458539962769,
  'miss_rate': 0.013958333060145378,
  'fall_out': 0.013854166492819786,
  'mcc': 0.9721875190734863},
 'test_evaluation': {'loss': 0.13638216257095337,
  'accuracy': 0.9570833444595337,
  'recall': 0.9579166769981384,
  'precision': 0.9571190476417542,
  'AUROC': 0.9905983209609985,
  'AUPRC': 0.9889681339263916,
  'f1_score': 0.957517683506012,
  'balanced_accuracy': 0.9574999809265137,
  'specificity': 0.9570833444595337,
  'miss_rate': 0.04208333417773247,
  'fall_out': 0.042916666716337204,
  'mcc': 0.915000319480896}}

Confusion Matrix of Cards CNN¶

In [116]:
holdout_to_plot = 0

true_labels = tf.math.argmax(CNN_holdout_test_labels[holdout_to_plot], axis=1)
predicted_labels = tf.math.argmax(CNN_holdout_predictions[holdout_to_plot], axis=1)

confusion_matrix = tf.math.confusion_matrix(
    true_labels,
    predicted_labels,
    num_classes=2,
    weights=None,
    name=None
)

sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set(rc = {'figure.figsize':(5,5)})
plt.xlabel('Predicted Labels', fontsize=10)
plt.ylabel('True Labels', fontsize=10)
plt.yticks(np.arange(2)+0.5,('Red','Yellow'), fontsize="10")
plt.xticks(np.arange(2)+0.5,('Red','Yellow'), fontsize="10" )
plt.show()
No description has been provided for this image

Generic soccer images classification¶

To distinguish between generic soccer images and soccer images belonging to a particular event a CNN is designed.
The model takes as input event and generic soccer images and considers the generic soccer event as an additional class.

In [24]:
epochs = 100
batch_size = 32
In [25]:
#data_soccer is a dict containing ['events'...] so group together classes to evaluate
data_soccer_with_generic_all = []
for event in data_soccer:
    for element in data_soccer[event]:
        data_soccer_with_generic_all.append(element)
        
for element in data_generic_events["Soccer"]:
    data_soccer_with_generic_all.append(element)

print(f"Data shape: {np.array(data_soccer_with_generic_all).shape}")
print(f"Labels shape: {labels_soccer_with_generic.shape}")
Data shape: (55199, 80, 80)
Labels shape: (55199, 10)
In [26]:
data_soccer_with_generic_all = [np.expand_dims(x, axis=-1) for x in data_soccer_with_generic_all]
In [27]:
def build_soccer_CNN_with_generic(
    input_shape: tuple
) -> tf.keras.Model:
    """ Returns the generic soccer CNN model.

    Parameters
    ------------------------
    input_shape: tuple
        The shape of the input.

    Returns
    ------------------------
    model: model
        The generic soccer model."""

    CNN = Sequential(name="Soccer_with_generic_CNN")
    CNN.add(layers.Input(input_shape))
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 256,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 256,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),

    CNN.add(layers.Flatten())
    
    for i in range(2):
        CNN.add(layers.Dense(units = 128, activation='relu'))
        CNN.add(layers.Dropout(0.5))
        
    CNN.add(layers.Dense(10, activation='softmax'))
    
    CNN.compile(
        optimizer='adam',
        loss='categorical_crossentropy',
        metrics=get_minimal_multiclass_metrics()
    )
    
    CNN.summary()
    
    return CNN

Main training loop - Soccer CNN + generic event¶

  • Normalize images' values in 0-1 range.
  • Build and train the model over the training set and test it over the test set for the different holdouts.
In [68]:
print("---- Soccer+generic CNN ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_with_generic_all, labels_soccer_with_generic))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([data_soccer_with_generic_all[x]/255 for x in train_indices]), np.array([data_soccer_with_generic_all[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer_with_generic.iloc[train_indices], labels_soccer_with_generic.iloc[test_indices]

    # Build CNN
    CNN = build_soccer_CNN_with_generic(x_train[0].shape)

    print("- Training model:\n")
    CNN_holdout_metrics, CNN_holdout_history = train_model(
        CNN,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size
    )
    
    predictions = get_predictions(CNN, x_test, batch_size=batch_size)
    CNN_holdout_predictions.append(predictions)
    CNN_holdout_test_labels.append(y_test)

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(CNN)
---- Soccer+generic CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_with_generic_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_12 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_12 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_13 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_13 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_14 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_14 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 conv2d_15 (Conv2D)          (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_15 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_3 (Flatten)         (None, 6400)              0         
                                                                 
 dense_9 (Dense)             (None, 128)               819328    
                                                                 
 dropout_6 (Dropout)         (None, 128)               0         
                                                                 
 dense_10 (Dense)            (None, 128)               16512     
                                                                 
 dropout_7 (Dropout)         (None, 128)               0         
                                                                 
 dense_11 (Dense)            (None, 10)                1290      
                                                                 
=================================================================
Total params: 1,871,242
Trainable params: 1,871,242
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
1380/1380 [==============================] - 26s 18ms/step - loss: 1.3459 - accuracy: 0.4946 - recall: 0.3038 - precision: 0.7262 - AUROC: 0.8963 - AUPRC: 0.5650 - val_loss: 0.6595 - val_accuracy: 0.7771 - val_recall: 0.6816 - val_precision: 0.8659 - val_AUROC: 0.9761 - val_AUPRC: 0.8600
Epoch 2/100
1380/1380 [==============================] - 24s 17ms/step - loss: 0.7398 - accuracy: 0.7548 - recall: 0.6603 - precision: 0.8371 - AUROC: 0.9679 - AUPRC: 0.8323 - val_loss: 0.4211 - val_accuracy: 0.8646 - val_recall: 0.8136 - val_precision: 0.9103 - val_AUROC: 0.9896 - val_AUPRC: 0.9368
Epoch 3/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.5390 - accuracy: 0.8249 - recall: 0.7695 - precision: 0.8783 - AUROC: 0.9815 - AUPRC: 0.9021 - val_loss: 0.3836 - val_accuracy: 0.8788 - val_recall: 0.8344 - val_precision: 0.9200 - val_AUROC: 0.9908 - val_AUPRC: 0.9469
Epoch 4/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.4291 - accuracy: 0.8608 - recall: 0.8230 - precision: 0.9015 - AUROC: 0.9874 - AUPRC: 0.9333 - val_loss: 0.2878 - val_accuracy: 0.9064 - val_recall: 0.8878 - val_precision: 0.9291 - val_AUROC: 0.9944 - val_AUPRC: 0.9661
Epoch 5/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3716 - accuracy: 0.8809 - recall: 0.8529 - precision: 0.9134 - AUROC: 0.9900 - AUPRC: 0.9479 - val_loss: 0.3053 - val_accuracy: 0.9010 - val_recall: 0.8820 - val_precision: 0.9232 - val_AUROC: 0.9930 - val_AUPRC: 0.9624
Epoch 6/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3212 - accuracy: 0.8989 - recall: 0.8751 - precision: 0.9262 - AUROC: 0.9920 - AUPRC: 0.9589 - val_loss: 0.2412 - val_accuracy: 0.9195 - val_recall: 0.9055 - val_precision: 0.9356 - val_AUROC: 0.9951 - val_AUPRC: 0.9744
Epoch 7/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.2869 - accuracy: 0.9090 - recall: 0.8898 - precision: 0.9333 - AUROC: 0.9933 - AUPRC: 0.9663 - val_loss: 0.2319 - val_accuracy: 0.9250 - val_recall: 0.9089 - val_precision: 0.9414 - val_AUROC: 0.9953 - val_AUPRC: 0.9761
Epoch 8/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.2553 - accuracy: 0.9187 - recall: 0.9019 - precision: 0.9399 - AUROC: 0.9944 - AUPRC: 0.9723 - val_loss: 0.2204 - val_accuracy: 0.9289 - val_recall: 0.9183 - val_precision: 0.9405 - val_AUROC: 0.9950 - val_AUPRC: 0.9776
Epoch 9/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.2392 - accuracy: 0.9240 - recall: 0.9091 - precision: 0.9435 - AUROC: 0.9948 - AUPRC: 0.9748 - val_loss: 0.1992 - val_accuracy: 0.9349 - val_recall: 0.9253 - val_precision: 0.9496 - val_AUROC: 0.9961 - val_AUPRC: 0.9809
Epoch 10/100
1380/1380 [==============================] - 26s 19ms/step - loss: 0.2084 - accuracy: 0.9332 - recall: 0.9201 - precision: 0.9492 - AUROC: 0.9959 - AUPRC: 0.9800 - val_loss: 0.2128 - val_accuracy: 0.9364 - val_recall: 0.9292 - val_precision: 0.9449 - val_AUROC: 0.9946 - val_AUPRC: 0.9776
Epoch 11/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.1975 - accuracy: 0.9375 - recall: 0.9254 - precision: 0.9530 - AUROC: 0.9960 - AUPRC: 0.9815 - val_loss: 0.2290 - val_accuracy: 0.9334 - val_recall: 0.9265 - val_precision: 0.9416 - val_AUROC: 0.9936 - val_AUPRC: 0.9748
345/345 [==============================] - 2s 6ms/step - loss: 0.2290 - accuracy: 0.9334 - recall: 0.9265 - precision: 0.9416 - AUROC: 0.9936 - AUPRC: 0.9748
345/345 [==============================] - 2s 4ms/step
1it [04:49, 289.95s/it]
-- HOLDOUT 2
Model: "Soccer_with_generic_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_16 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_16 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_17 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_17 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_18 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_18 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 conv2d_19 (Conv2D)          (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_19 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_4 (Flatten)         (None, 6400)              0         
                                                                 
 dense_12 (Dense)            (None, 128)               819328    
                                                                 
 dropout_8 (Dropout)         (None, 128)               0         
                                                                 
 dense_13 (Dense)            (None, 128)               16512     
                                                                 
 dropout_9 (Dropout)         (None, 128)               0         
                                                                 
 dense_14 (Dense)            (None, 10)                1290      
                                                                 
=================================================================
Total params: 1,871,242
Trainable params: 1,871,242
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
1380/1380 [==============================] - 27s 19ms/step - loss: 1.3777 - accuracy: 0.4855 - recall: 0.2880 - precision: 0.7169 - AUROC: 0.8912 - AUPRC: 0.5474 - val_loss: 0.7099 - val_accuracy: 0.7647 - val_recall: 0.6688 - val_precision: 0.8466 - val_AUROC: 0.9722 - val_AUPRC: 0.8428
Epoch 2/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.7676 - accuracy: 0.7400 - recall: 0.6429 - precision: 0.8255 - AUROC: 0.9659 - AUPRC: 0.8200 - val_loss: 0.4444 - val_accuracy: 0.8515 - val_recall: 0.8079 - val_precision: 0.9020 - val_AUROC: 0.9885 - val_AUPRC: 0.9326
Epoch 3/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.5745 - accuracy: 0.8123 - recall: 0.7530 - precision: 0.8703 - AUROC: 0.9794 - AUPRC: 0.8919 - val_loss: 0.3906 - val_accuracy: 0.8772 - val_recall: 0.8272 - val_precision: 0.9238 - val_AUROC: 0.9917 - val_AUPRC: 0.9489
Epoch 4/100
1380/1380 [==============================] - 26s 19ms/step - loss: 0.4593 - accuracy: 0.8507 - recall: 0.8098 - precision: 0.8952 - AUROC: 0.9859 - AUPRC: 0.9256 - val_loss: 0.3006 - val_accuracy: 0.9040 - val_recall: 0.8798 - val_precision: 0.9297 - val_AUROC: 0.9940 - val_AUPRC: 0.9643
Epoch 5/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3975 - accuracy: 0.8711 - recall: 0.8391 - precision: 0.9094 - AUROC: 0.9890 - AUPRC: 0.9424 - val_loss: 0.2791 - val_accuracy: 0.9138 - val_recall: 0.8895 - val_precision: 0.9348 - val_AUROC: 0.9942 - val_AUPRC: 0.9682
Epoch 6/100
1380/1380 [==============================] - 25s 18ms/step - loss: 0.3414 - accuracy: 0.8899 - recall: 0.8635 - precision: 0.9200 - AUROC: 0.9914 - AUPRC: 0.9551 - val_loss: 0.2998 - val_accuracy: 0.9024 - val_recall: 0.8793 - val_precision: 0.9294 - val_AUROC: 0.9935 - val_AUPRC: 0.9655
Epoch 7/100
1380/1380 [==============================] - 31s 23ms/step - loss: 0.3008 - accuracy: 0.9036 - recall: 0.8820 - precision: 0.9320 - AUROC: 0.9930 - AUPRC: 0.9638 - val_loss: 0.2484 - val_accuracy: 0.9255 - val_recall: 0.9132 - val_precision: 0.9399 - val_AUROC: 0.9941 - val_AUPRC: 0.9720
Epoch 8/100
1380/1380 [==============================] - 44s 32ms/step - loss: 0.2664 - accuracy: 0.9151 - recall: 0.8959 - precision: 0.9385 - AUROC: 0.9942 - AUPRC: 0.9704 - val_loss: 0.2605 - val_accuracy: 0.9164 - val_recall: 0.8975 - val_precision: 0.9384 - val_AUROC: 0.9946 - val_AUPRC: 0.9713
Epoch 9/100
1380/1380 [==============================] - 45s 32ms/step - loss: 0.2508 - accuracy: 0.9184 - recall: 0.9009 - precision: 0.9401 - AUROC: 0.9945 - AUPRC: 0.9727 - val_loss: 0.2333 - val_accuracy: 0.9286 - val_recall: 0.9193 - val_precision: 0.9419 - val_AUROC: 0.9943 - val_AUPRC: 0.9756
Epoch 10/100
1380/1380 [==============================] - 45s 33ms/step - loss: 0.2262 - accuracy: 0.9268 - recall: 0.9118 - precision: 0.9458 - AUROC: 0.9954 - AUPRC: 0.9773 - val_loss: 0.2288 - val_accuracy: 0.9299 - val_recall: 0.9201 - val_precision: 0.9428 - val_AUROC: 0.9943 - val_AUPRC: 0.9761
Epoch 11/100
1380/1380 [==============================] - 45s 33ms/step - loss: 0.2143 - accuracy: 0.9320 - recall: 0.9173 - precision: 0.9504 - AUROC: 0.9955 - AUPRC: 0.9789 - val_loss: 0.2264 - val_accuracy: 0.9313 - val_recall: 0.9234 - val_precision: 0.9439 - val_AUROC: 0.9945 - val_AUPRC: 0.9773
Epoch 12/100
1380/1380 [==============================] - 42s 30ms/step - loss: 0.1942 - accuracy: 0.9378 - recall: 0.9236 - precision: 0.9552 - AUROC: 0.9963 - AUPRC: 0.9821 - val_loss: 0.1948 - val_accuracy: 0.9439 - val_recall: 0.9358 - val_precision: 0.9529 - val_AUROC: 0.9950 - val_AUPRC: 0.9805
Epoch 13/100
1380/1380 [==============================] - 44s 32ms/step - loss: 0.1889 - accuracy: 0.9409 - recall: 0.9283 - precision: 0.9567 - AUROC: 0.9961 - AUPRC: 0.9828 - val_loss: 0.2340 - val_accuracy: 0.9294 - val_recall: 0.9204 - val_precision: 0.9412 - val_AUROC: 0.9937 - val_AUPRC: 0.9753
Epoch 14/100
1380/1380 [==============================] - 43s 31ms/step - loss: 0.1748 - accuracy: 0.9436 - recall: 0.9326 - precision: 0.9593 - AUROC: 0.9967 - AUPRC: 0.9851 - val_loss: 0.2211 - val_accuracy: 0.9364 - val_recall: 0.9294 - val_precision: 0.9471 - val_AUROC: 0.9938 - val_AUPRC: 0.9767
345/345 [==============================] - 3s 10ms/step - loss: 0.2211 - accuracy: 0.9364 - recall: 0.9294 - precision: 0.9471 - AUROC: 0.9938 - AUPRC: 0.9767
345/345 [==============================] - 2s 6ms/step
2it [13:13, 396.60s/it]
In [69]:
print(f"Soccer+Generic CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : test - {CNN_metrics[0]['test_evaluation']['accuracy']}")
print(f"AUROC : test - {CNN_metrics[0]['test_evaluation']['AUROC']}")
print(f"AUPRC : test - {CNN_metrics[0]['test_evaluation']['AUPRC']}")
print("-"*80)
CNN Metrics - 2-holdouts estimate:
Accuracy : test - 0.9334239363670349
AUROC : test - 0.9935887455940247
AUPRC : test - 0.9748353958129883
--------------------------------------------------------------------------------

Confusion Matrix - Soccer CNN + generic event¶

Note that the generic class has fewer examples and only 206 test samples.

In [75]:
true_labels = tf.math.argmax(CNN_holdout_predictions[1], axis=1)
predicted_labels = tf.math.argmax(CNN_holdout_test_labels[1], axis=1)

confusion_matrix = tf.math.confusion_matrix(
    true_labels,
    predicted_labels,
    num_classes=10,
    weights=None,
    name=None
)

sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(10,10)})
plt.xlabel('Predicted Labels', fontsize=14)
plt.ylabel('True Labels', fontsize=14)
plt.yticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11")
plt.xticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11" )
plt.show()
No description has been provided for this image
In [73]:
# Save Soccer CNN with generic class
soccerAndGeneric_model = CNN_models[1]
soccerAndGeneric_model.save_weights('modelWeights/soccerAndGeneric_model.h5')
In [72]:
# Soccer+generic CNN metrics
CNN_metrics[1]
Out[72]:
{'test_evaluation': {'loss': 0.22106623649597168,
  'accuracy': 0.936413049697876,
  'recall': 0.9294384121894836,
  'precision': 0.9471109509468079,
  'AUROC': 0.9938239455223083,
  'AUPRC': 0.976667046546936}}
In [79]:
# Save results (save dictionary to soccer_with_generic_metrics.pkl file)
with open('results/soccer_with_generic_metrics.pkl', 'wb') as fp:
    pickle.dump(CNN_metrics, fp)
    print('dictionary saved successfully to file')
dictionary saved successfully to file

Evaluation on real dataset¶

In [46]:
image_shape_gray = (soccer_images_dim,soccer_images_dim,1)
image_shape_rgb = (soccer_images_dim,soccer_images_dim,3)
In [76]:
## Load Model
# Create models
soccer_events_and_generic = build_soccer_CNN_with_generic(image_shape_gray)

# Restore the weights
soccer_events_and_generic.load_weights('modelWeights/soccerAndGeneric_model.h5')
soccer_events_and_generic._name = "soccer_events_and_generic"
Model: "Soccer_with_generic_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_20 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_20 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_21 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_21 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_22 (Conv2D)          (None, 20, 20, 256)       295168    
                                                                 
 max_pooling2d_22 (MaxPoolin  (None, 10, 10, 256)      0         
 g2D)                                                            
                                                                 
 conv2d_23 (Conv2D)          (None, 10, 10, 256)       590080    
                                                                 
 max_pooling2d_23 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_5 (Flatten)         (None, 6400)              0         
                                                                 
 dense_15 (Dense)            (None, 128)               819328    
                                                                 
 dropout_10 (Dropout)        (None, 128)               0         
                                                                 
 dense_16 (Dense)            (None, 128)               16512     
                                                                 
 dropout_11 (Dropout)        (None, 128)               0         
                                                                 
 dense_17 (Dense)            (None, 10)                1290      
                                                                 
=================================================================
Total params: 1,871,242
Trainable params: 1,871,242
Non-trainable params: 0
_________________________________________________________________
In [61]:
folderMyTestDataset = r'D:\MachineLearning\Datasets\Soccer\myDataset' # RealSoccer + events and genericSoccer images

data_myTest_t, _ = loadData(folderMyTestDataset, False)
In [62]:
soccer_generic_data = []

for folder in data_myTest_t:
    if folder != "Events" and folder != "Generic":
        for i in range(len(data_myTest_t[folder])):
            soccer_generic_data.append(data_myTest_t[folder][i])

for folder in data_myTest_t:
    if folder == "Generic":
        for i in range(len(data_myTest_t[folder])):
            soccer_generic_data.append(data_myTest_t[folder][i])
In [63]:
soccer_generic_data = np.array([np.expand_dims(x/255, axis=-1) for x in soccer_generic_data])
In [64]:
print(f"Data shape: {soccer_generic_data.shape}")
print(f"Labels shape: {soccer_generic_labels.shape}")
Data shape: (100, 80, 80, 1)
Labels shape: (100, 10)
In [77]:
soccer_generic_predictions = get_predictions(soccer_events_and_generic, soccer_generic_data, 2)
          
# Shape (n_samples,)
labels = np.argmax(soccer_generic_labels, axis=1)
predictions = np.argmax(soccer_generic_predictions, axis=1)

acc, auroc, auprc = get_metrics(
    labels = labels,
    predictions = predictions
)

print(f"{soccer_events_and_generic.name} metrics on real dataset:")
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*80)

true_labels = labels
predicted_labels = predictions

confusion_matrix = tf.math.confusion_matrix(
    true_labels,
    predicted_labels,
    num_classes=10,
    weights=None,
    name=None
)

sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(8,8)})
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.yticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11")
plt.xticks(np.arange(10)+0.5,('Cards','Center','Corner', 'Free Kick', 'Left', 'Penalty', 'Right', 'Tackle', 'Substitute', 'Generic'), fontsize="11" )
plt.show()
50/50 [==============================] - 0s 5ms/step
soccer_events_and_generic metrics on real dataset:
Accuracy: 0.62
------------------------------
AUROC: 0.7888888888888889
------------------------------
AUPRC: 0.45661860361860357
--------------------------------------------------------------------------------
No description has been provided for this image

Complete classification pipeline definition¶

So far a few models were developed soccer event detection:

  • Soccer CNN
  • Soccer CNN with Data Augmentation
  • ResNet
  • VGG16
  • Soccer CNN + generic event

The basic soccer CNN and soccer CNN with generic class show the best results under these specific test conditions.

A model for Cards binary classification was also deployed:

  • Cards CNN

Here is a comparison of the various models performances on the event soccer detection task.

In [1]:
results = {"model":[], "accuracy":[], "AUROC":[], "AUPRC":[]}
In [4]:
# Load Results (Read dictionary pkl file)
with open('results/basic_CNN_metrics.pkl', 'rb') as fp:
    basic_CNN_metrics = pickle.load(fp)
with open('results/data_augmentation_basic_CNN_metrics.pkl', 'rb') as fp:
    data_augmentation_basic_CNN_metrics = pickle.load(fp)
with open('results/resNet_gray_metrics.pkl', 'rb') as fp:
    resNet_gray_metrics = pickle.load(fp)
with open('results/resNet_rgb_metrics.pkl', 'rb') as fp:
    resNet_rgb_metrics = pickle.load(fp)
with open('results/vgg_gray_metrics.pkl', 'rb') as fp:
    vgg_gray_metrics = pickle.load(fp)
with open('results/vgg_rgb_metrics.pkl', 'rb') as fp:
    vgg_rgb_metrics = pickle.load(fp)
with open('results/soccer_with_generic_metrics.pkl', 'rb') as fp:
    soccer_with_generic_metrics = []
    soccer_with_generic_metrics.append(pickle.load(fp))
In [6]:
# CNN metrics
for metrics, model in ((basic_CNN_metrics, "basic CNN"), (data_augmentation_basic_CNN_metrics, "data augmentation basic CNN"), (resNet_gray_metrics, "resNet gray CNN"), (resNet_rgb_metrics, "resNet rgb CNN"), (vgg_gray_metrics, "vgg gray CNN"), (vgg_rgb_metrics, "vgg rgb CNN"), (soccer_with_generic_metrics, "soccer with generic CNN")):
    i = 0
    for holdout in metrics:
        results["model"].append(model)
        results["accuracy"].append(holdout["test_evaluation"]["accuracy"])
        results["AUROC"].append(holdout["test_evaluation"]["AUROC"])
        results["AUPRC"].append(holdout["test_evaluation"]["AUPRC"])
In [8]:
import pandas as pd
In [9]:
results_df = pd.DataFrame(results)
In [ ]:
#TODO make nice graph
In [17]:
from barplots import barplots

barplots(
    results_df,
    groupby=["model"],
    orientation="horizontal",
    height=12,
    legend_position="upper left"
)
                                                                                                                       
Out[17]:
[(<Figure size 2400x420 with 1 Axes>,
  array([<Axes: title={'center': 'AUROC'}, xlabel='AUROC'>], dtype=object)),
 (<Figure size 2400x420 with 1 Axes>,
  array([<Axes: title={'center': 'AUPRC'}, xlabel='AUPRC'>], dtype=object)),
 (<Figure size 2400x420 with 1 Axes>,
  array([<Axes: title={'center': 'Accuracy'}, xlabel='Accuracy'>],
        dtype=object))]
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image

The next task to tackle is:

  • Random image classification: filter out non soccer-related images.

This problem is studied with two approaches, generiting two different classification pipelines.

Approach A: Supervised Learning
Soccer VS Random classification CNN > Soccer Event Classification CNN > Cards Classification CNN.

Approach B: Unupervised Learning
Soccer VS Random clustering > Soccer Event Classification CNN > Cards Classification CNN.

Approach A - Supervised Learning¶

Design a CNN model for the binary classification on soccer related images and random images.

In [26]:
number_of_splits = 2

holdouts_generator = StratifiedShuffleSplit(
    n_splits = number_of_splits,
    test_size = 0.2,
    random_state = 42
)
In [27]:
epochs = 100
batch_size = 32
In [29]:
# Create dataset (random soccer samples and event samples)
data_soccer_vs_events = []
labels_soccer_vs_events = []

# 1000 soccer images from soccer events
data_soccer_events = [np.squeeze(x, axis=-1) for x in data_soccer_all]
soccer_elements = random.sample(data_soccer_events, 1000)
data_soccer_vs_events.extend(soccer_elements)

# 400 soccer images from general soccer
general_soccer_elements = random.sample(data_generic_events["Soccer"], 400)
data_soccer_vs_events.extend(general_soccer_elements)

# 1400 images from generic events
data_soccer_vs_events.extend(data_generic_events["Random"])

for i in range(1400):
    labels_soccer_vs_events.append(0)
    
for i in range(1400):
    labels_soccer_vs_events.append(1)
    
labels_soccer_vs_events = one_hot_encoding(labels_soccer_vs_events, 2)
        

print(f"Data shape: {np.array(data_soccer_vs_events).shape}")
print(f"Labels shape: {labels_soccer_vs_events.shape}")
Data shape: (2800, 80, 80)
Labels shape: (2800, 2)
In [30]:
data_soccer_vs_events = [np.expand_dims(x, axis=-1) for x in data_soccer_vs_events]
In [31]:
def build_soccer_nonSoccer_CNN(
    input_shape: tuple
) -> tf.keras.Model:
    """ Returns the basic soccer CNN model.

    Parameters
    ------------------------
    input_shape: tuple
        The shape of the input.

    Returns
    ------------------------
    model: model
        The soccer vs random model."""

    CNN = Sequential(name="Soccer_Events_CNN")
    CNN.add(layers.Input(input_shape))
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 128,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),
    
    CNN.add(layers.Conv2D(
        filters = 256,
        kernel_size = 3,
        padding='same',
        activation='relu'))
    CNN.add(layers.MaxPooling2D()),

    CNN.add(layers.Flatten())
    
    for i in range(2):
        CNN.add(layers.Dense(units = 128, activation='relu'))
        CNN.add(layers.Dropout(0.5))
        
    CNN.add(layers.Dense(2, activation='softmax'))
    
    CNN.compile(
        optimizer='adam',
        loss='binary_crossentropy',
        metrics=get_standard_binary_metrics()
    )
    
    CNN.summary()
    
    return CNN

Main training loop - Soccer vs Non-Soccer CNN¶

  • Normalize images' values in 0-1 range.
  • Build and train the model over the training set and test it over the test set for the different holdouts.
In [39]:
print("---- Soccer vs Non-Soccer CNN ----")

CNN_metrics = []
CNN_history = []

CNN_holdout_predictions = []
CNN_holdout_test_labels = []

CNN_models = []

# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(tqdm(holdouts_generator.split(data_soccer_vs_events, labels_soccer_vs_events))):
    print(f"-- HOLDOUT {holdout_number+1}")
    # Train/Test data + Normalization
    x_train, x_test = np.array([data_soccer_vs_events[x]/255 for x in train_indices]), np.array([data_soccer_vs_events[x]/255 for x in test_indices])
    y_train, y_test = labels_soccer_vs_events.iloc[train_indices], labels_soccer_vs_events.iloc[test_indices]

    # Build CNN
    CNN = build_soccer_nonSoccer_CNN(x_train[0].shape)

    print("- Training model:\n")
    CNN_holdout_metrics, CNN_holdout_history = train_model(
        CNN,
        x_train,
        x_test,
        y_train.values,
        y_test.values,
        epochs,
        batch_size
    )
    
    predictions = get_predictions(CNN, x_test, batch_size=batch_size)
    CNN_holdout_predictions.append(predictions)
    CNN_holdout_test_labels.append(y_test)

    CNN_metrics.append(CNN_holdout_metrics)
    CNN_history.append(CNN_holdout_history)
    
    CNN_models.append(CNN)
---- Soccer vs Non-Soccer CNN ----
0it [00:00, ?it/s]
-- HOLDOUT 1
Model: "Soccer_Events_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_16 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_16 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_17 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_17 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_18 (Conv2D)          (None, 20, 20, 128)       147584    
                                                                 
 max_pooling2d_18 (MaxPoolin  (None, 10, 10, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_19 (Conv2D)          (None, 10, 10, 256)       295168    
                                                                 
 max_pooling2d_19 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_4 (Flatten)         (None, 6400)              0         
                                                                 
 dense_12 (Dense)            (None, 128)               819328    
                                                                 
 dropout_8 (Dropout)         (None, 128)               0         
                                                                 
 dense_13 (Dense)            (None, 128)               16512     
                                                                 
 dropout_9 (Dropout)         (None, 128)               0         
                                                                 
 dense_14 (Dense)            (None, 2)                 258       
                                                                 
=================================================================
Total params: 1,427,714
Trainable params: 1,427,714
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
70/70 [==============================] - 3s 22ms/step - loss: 0.6000 - accuracy: 0.6621 - recall: 0.6621 - precision: 0.6621 - AUROC: 0.7414 - AUPRC: 0.7439 - f1_score: 0.6621 - balanced_accuracy: 0.6621 - specificity: 0.6621 - miss_rate: 0.3379 - fall_out: 0.3379 - mcc: 0.3241 - val_loss: 0.3223 - val_accuracy: 0.8679 - val_recall: 0.8679 - val_precision: 0.8679 - val_AUROC: 0.9379 - val_AUPRC: 0.9351 - val_f1_score: 0.8679 - val_balanced_accuracy: 0.8679 - val_specificity: 0.8679 - val_miss_rate: 0.1321 - val_fall_out: 0.1321 - val_mcc: 0.7357
Epoch 2/100
70/70 [==============================] - 1s 16ms/step - loss: 0.2208 - accuracy: 0.9170 - recall: 0.9170 - precision: 0.9170 - AUROC: 0.9696 - AUPRC: 0.9657 - f1_score: 0.9170 - balanced_accuracy: 0.9170 - specificity: 0.9170 - miss_rate: 0.0830 - fall_out: 0.0830 - mcc: 0.8339 - val_loss: 0.1392 - val_accuracy: 0.9464 - val_recall: 0.9464 - val_precision: 0.9464 - val_AUROC: 0.9898 - val_AUPRC: 0.9893 - val_f1_score: 0.9464 - val_balanced_accuracy: 0.9464 - val_specificity: 0.9464 - val_miss_rate: 0.0536 - val_fall_out: 0.0536 - val_mcc: 0.8929
Epoch 3/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1640 - accuracy: 0.9415 - recall: 0.9415 - precision: 0.9415 - AUROC: 0.9819 - AUPRC: 0.9790 - f1_score: 0.9415 - balanced_accuracy: 0.9415 - specificity: 0.9415 - miss_rate: 0.0585 - fall_out: 0.0585 - mcc: 0.8830 - val_loss: 0.1067 - val_accuracy: 0.9607 - val_recall: 0.9607 - val_precision: 0.9607 - val_AUROC: 0.9926 - val_AUPRC: 0.9921 - val_f1_score: 0.9607 - val_balanced_accuracy: 0.9607 - val_specificity: 0.9607 - val_miss_rate: 0.0393 - val_fall_out: 0.0393 - val_mcc: 0.9214
Epoch 4/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1273 - accuracy: 0.9580 - recall: 0.9580 - precision: 0.9580 - AUROC: 0.9870 - AUPRC: 0.9844 - f1_score: 0.9580 - balanced_accuracy: 0.9580 - specificity: 0.9580 - miss_rate: 0.0420 - fall_out: 0.0420 - mcc: 0.9161 - val_loss: 0.1484 - val_accuracy: 0.9536 - val_recall: 0.9536 - val_precision: 0.9536 - val_AUROC: 0.9781 - val_AUPRC: 0.9727 - val_f1_score: 0.9536 - val_balanced_accuracy: 0.9536 - val_specificity: 0.9536 - val_miss_rate: 0.0464 - val_fall_out: 0.0464 - val_mcc: 0.9071
Epoch 5/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1297 - accuracy: 0.9527 - recall: 0.9527 - precision: 0.9527 - AUROC: 0.9848 - AUPRC: 0.9818 - f1_score: 0.9527 - balanced_accuracy: 0.9527 - specificity: 0.9527 - miss_rate: 0.0473 - fall_out: 0.0473 - mcc: 0.9054 - val_loss: 0.1183 - val_accuracy: 0.9696 - val_recall: 0.9696 - val_precision: 0.9696 - val_AUROC: 0.9817 - val_AUPRC: 0.9765 - val_f1_score: 0.9696 - val_balanced_accuracy: 0.9696 - val_specificity: 0.9696 - val_miss_rate: 0.0304 - val_fall_out: 0.0304 - val_mcc: 0.9393
70/70 [==============================] - 0s 6ms/step - loss: 0.0714 - accuracy: 0.9741 - recall: 0.9741 - precision: 0.9741 - AUROC: 0.9941 - AUPRC: 0.9929 - f1_score: 0.9741 - balanced_accuracy: 0.9741 - specificity: 0.9741 - miss_rate: 0.0259 - fall_out: 0.0259 - mcc: 0.9482
18/18 [==============================] - 0s 6ms/step - loss: 0.1183 - accuracy: 0.9696 - recall: 0.9696 - precision: 0.9696 - AUROC: 0.9817 - AUPRC: 0.9765 - f1_score: 0.9696 - balanced_accuracy: 0.9696 - specificity: 0.9696 - miss_rate: 0.0304 - fall_out: 0.0304 - mcc: 0.9393
18/18 [==============================] - 0s 4ms/step
1it [00:08,  8.43s/it]
-- HOLDOUT 2
Model: "Soccer_Events_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_20 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_20 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_21 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_21 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_22 (Conv2D)          (None, 20, 20, 128)       147584    
                                                                 
 max_pooling2d_22 (MaxPoolin  (None, 10, 10, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_23 (Conv2D)          (None, 10, 10, 256)       295168    
                                                                 
 max_pooling2d_23 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_5 (Flatten)         (None, 6400)              0         
                                                                 
 dense_15 (Dense)            (None, 128)               819328    
                                                                 
 dropout_10 (Dropout)        (None, 128)               0         
                                                                 
 dense_16 (Dense)            (None, 128)               16512     
                                                                 
 dropout_11 (Dropout)        (None, 128)               0         
                                                                 
 dense_17 (Dense)            (None, 2)                 258       
                                                                 
=================================================================
Total params: 1,427,714
Trainable params: 1,427,714
Non-trainable params: 0
_________________________________________________________________
- Training model:

Epoch 1/100
70/70 [==============================] - 2s 22ms/step - loss: 0.5652 - accuracy: 0.6772 - recall: 0.6772 - precision: 0.6772 - AUROC: 0.7733 - AUPRC: 0.7796 - f1_score: 0.6772 - balanced_accuracy: 0.6772 - specificity: 0.6772 - miss_rate: 0.3228 - fall_out: 0.3228 - mcc: 0.3545 - val_loss: 0.1876 - val_accuracy: 0.9286 - val_recall: 0.9286 - val_precision: 0.9286 - val_AUROC: 0.9778 - val_AUPRC: 0.9763 - val_f1_score: 0.9286 - val_balanced_accuracy: 0.9286 - val_specificity: 0.9286 - val_miss_rate: 0.0714 - val_fall_out: 0.0714 - val_mcc: 0.8571
Epoch 2/100
70/70 [==============================] - 1s 16ms/step - loss: 0.2142 - accuracy: 0.9246 - recall: 0.9246 - precision: 0.9246 - AUROC: 0.9730 - AUPRC: 0.9700 - f1_score: 0.9246 - balanced_accuracy: 0.9246 - specificity: 0.9246 - miss_rate: 0.0754 - fall_out: 0.0754 - mcc: 0.8491 - val_loss: 0.1192 - val_accuracy: 0.9518 - val_recall: 0.9518 - val_precision: 0.9518 - val_AUROC: 0.9934 - val_AUPRC: 0.9936 - val_f1_score: 0.9518 - val_balanced_accuracy: 0.9518 - val_specificity: 0.9518 - val_miss_rate: 0.0482 - val_fall_out: 0.0482 - val_mcc: 0.9036
Epoch 3/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1478 - accuracy: 0.9509 - recall: 0.9509 - precision: 0.9509 - AUROC: 0.9840 - AUPRC: 0.9812 - f1_score: 0.9509 - balanced_accuracy: 0.9509 - specificity: 0.9509 - miss_rate: 0.0491 - fall_out: 0.0491 - mcc: 0.9018 - val_loss: 0.1293 - val_accuracy: 0.9571 - val_recall: 0.9571 - val_precision: 0.9571 - val_AUROC: 0.9764 - val_AUPRC: 0.9698 - val_f1_score: 0.9571 - val_balanced_accuracy: 0.9571 - val_specificity: 0.9571 - val_miss_rate: 0.0429 - val_fall_out: 0.0429 - val_mcc: 0.9143
Epoch 4/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1589 - accuracy: 0.9406 - recall: 0.9406 - precision: 0.9406 - AUROC: 0.9817 - AUPRC: 0.9788 - f1_score: 0.9406 - balanced_accuracy: 0.9406 - specificity: 0.9406 - miss_rate: 0.0594 - fall_out: 0.0594 - mcc: 0.8813 - val_loss: 0.1174 - val_accuracy: 0.9536 - val_recall: 0.9536 - val_precision: 0.9536 - val_AUROC: 0.9862 - val_AUPRC: 0.9838 - val_f1_score: 0.9536 - val_balanced_accuracy: 0.9536 - val_specificity: 0.9536 - val_miss_rate: 0.0464 - val_fall_out: 0.0464 - val_mcc: 0.9071
Epoch 5/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1201 - accuracy: 0.9621 - recall: 0.9621 - precision: 0.9621 - AUROC: 0.9865 - AUPRC: 0.9835 - f1_score: 0.9621 - balanced_accuracy: 0.9621 - specificity: 0.9621 - miss_rate: 0.0379 - fall_out: 0.0379 - mcc: 0.9241 - val_loss: 0.1691 - val_accuracy: 0.9214 - val_recall: 0.9214 - val_precision: 0.9214 - val_AUROC: 0.9790 - val_AUPRC: 0.9766 - val_f1_score: 0.9214 - val_balanced_accuracy: 0.9214 - val_specificity: 0.9214 - val_miss_rate: 0.0786 - val_fall_out: 0.0786 - val_mcc: 0.8429
Epoch 6/100
70/70 [==============================] - 1s 16ms/step - loss: 0.1184 - accuracy: 0.9594 - recall: 0.9594 - precision: 0.9594 - AUROC: 0.9893 - AUPRC: 0.9875 - f1_score: 0.9594 - balanced_accuracy: 0.9594 - specificity: 0.9594 - miss_rate: 0.0406 - fall_out: 0.0406 - mcc: 0.9187 - val_loss: 0.1247 - val_accuracy: 0.9554 - val_recall: 0.9554 - val_precision: 0.9554 - val_AUROC: 0.9840 - val_AUPRC: 0.9805 - val_f1_score: 0.9554 - val_balanced_accuracy: 0.9554 - val_specificity: 0.9554 - val_miss_rate: 0.0446 - val_fall_out: 0.0446 - val_mcc: 0.9107
70/70 [==============================] - 0s 6ms/step - loss: 0.0956 - accuracy: 0.9670 - recall: 0.9670 - precision: 0.9670 - AUROC: 0.9893 - AUPRC: 0.9867 - f1_score: 0.9670 - balanced_accuracy: 0.9670 - specificity: 0.9670 - miss_rate: 0.0330 - fall_out: 0.0330 - mcc: 0.9339
18/18 [==============================] - 0s 6ms/step - loss: 0.1247 - accuracy: 0.9554 - recall: 0.9554 - precision: 0.9554 - AUROC: 0.9840 - AUPRC: 0.9805 - f1_score: 0.9554 - balanced_accuracy: 0.9554 - specificity: 0.9554 - miss_rate: 0.0446 - fall_out: 0.0446 - mcc: 0.9107
18/18 [==============================] - 0s 4ms/step
2it [00:17,  8.68s/it]
In [40]:
CNN_metrics_estimate = model_metrics_holdout_estimate(CNN_metrics, number_of_splits)

print(f"Soccer VS Events - CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']}  --  test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']}  --  test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']}  --  test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN - Train history:")
plot_train_history(CNN_history)
print("-"*100)
Soccer VS Events - CNN Metrics - 2-holdouts estimate:
Accuracy : train - 0.9705357253551483  --  test - 0.9625000059604645
AUROC : train - 0.9916850328445435  --  test - 0.982855498790741
AUPRC : train - 0.9897907674312592  --  test - 0.9784724414348602
--------------------------------------------------------------------------------
CNN - Train history:
No description has been provided for this image
No description has been provided for this image
----------------------------------------------------------------------------------------------------
In [42]:
# Soccer vs Non-Soccer CNN Metrics
CNN_metrics[0]
Out[42]:
{'train_evaluation': {'loss': 0.0713760033249855,
  'accuracy': 0.9741071462631226,
  'recall': 0.9741071462631226,
  'precision': 0.9741071462631226,
  'AUROC': 0.9940677881240845,
  'AUPRC': 0.9928805828094482,
  'f1_score': 0.9741071462631226,
  'balanced_accuracy': 0.9741071462631226,
  'specificity': 0.9741071462631226,
  'miss_rate': 0.02589285746216774,
  'fall_out': 0.02589285746216774,
  'mcc': 0.9482142925262451},
 'test_evaluation': {'loss': 0.11834557354450226,
  'accuracy': 0.9696428775787354,
  'recall': 0.9696428775787354,
  'precision': 0.9696428775787354,
  'AUROC': 0.9816644787788391,
  'AUPRC': 0.9764835834503174,
  'f1_score': 0.9696428775787354,
  'balanced_accuracy': 0.9696428775787354,
  'specificity': 0.9696428775787354,
  'miss_rate': 0.03035714291036129,
  'fall_out': 0.03035714291036129,
  'mcc': 0.9392856955528259}}
In [43]:
# Save Soccer CNN 
soccer_nonSoccer_model = CNN_models[0]
soccer_nonSoccer_model.save_weights('modelWeights/soccer_nonSoccer_model.h5')

Evaluation on myDataset¶

In [47]:
## Load Model
# Create models
events_vs_soccer = build_soccer_nonSoccer_CNN(image_shape_gray)

# Restore the weights
events_vs_soccer.load_weights('modelWeights/events_vs_soccer_model.h5')
events_vs_soccer._name = "events_vs_soccer"
Model: "Soccer_Events_CNN"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_24 (Conv2D)          (None, 80, 80, 128)       1280      
                                                                 
 max_pooling2d_24 (MaxPoolin  (None, 40, 40, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_25 (Conv2D)          (None, 40, 40, 128)       147584    
                                                                 
 max_pooling2d_25 (MaxPoolin  (None, 20, 20, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_26 (Conv2D)          (None, 20, 20, 128)       147584    
                                                                 
 max_pooling2d_26 (MaxPoolin  (None, 10, 10, 128)      0         
 g2D)                                                            
                                                                 
 conv2d_27 (Conv2D)          (None, 10, 10, 256)       295168    
                                                                 
 max_pooling2d_27 (MaxPoolin  (None, 5, 5, 256)        0         
 g2D)                                                            
                                                                 
 flatten_6 (Flatten)         (None, 6400)              0         
                                                                 
 dense_18 (Dense)            (None, 128)               819328    
                                                                 
 dropout_12 (Dropout)        (None, 128)               0         
                                                                 
 dense_19 (Dense)            (None, 128)               16512     
                                                                 
 dropout_13 (Dropout)        (None, 128)               0         
                                                                 
 dense_20 (Dense)            (None, 2)                 258       
                                                                 
=================================================================
Total params: 1,427,714
Trainable params: 1,427,714
Non-trainable params: 0
_________________________________________________________________
In [48]:
#data_soccer is a dict containing ['events'...] so group together classes to evaluate
data_myTest_all = []
for event in data_myTest:
    for element in data_myTest[event]:
        data_myTest_all.append(element)

print(f"Data shape: {np.array(data_myTest_all).shape}")
print(f"Labels shape: {labels_myTest.shape}")
Data shape: (110, 80, 80)
Labels shape: (110, 11)
In [49]:
data_myTest_all = np.array([np.expand_dims(x/255, axis=-1) for x in data_myTest_all])
In [50]:
# Adjust labels for each step (only for final evaluation and confrontation)
# Events VS Soccer
events_vs_soccer_labels = pd.DataFrame(columns=[0,1])

event_value = [0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]

for index, row in labels_myTest.iterrows():
    if np.array_equal(row.values, event_value):
        new_row = {0: 0.0, 1: 1.0}
        events_vs_soccer_labels.loc[index] = new_row
    else:
        new_row = {0: 1.0, 1: 0.0}
        events_vs_soccer_labels.loc[index] = new_row
        
events_vs_soccer_labels.sample(2)
Out[50]:
0 1
17 1.0 0.0
92 1.0 0.0
In [56]:
event_vs_soccer_predictions = get_predictions(events_vs_soccer, data_myTest_all, batch_size=2)
          
# Shape (n_samples,)
labels = np.argmax(events_vs_soccer_labels, axis=1)
predictions = np.argmax(event_vs_soccer_predictions, axis=1)

acc, auroc, auprc = get_metrics(
    labels = labels,
    predictions = predictions
)

print(f"{events_vs_soccer.name} metrics on real dataset:")
print(f"Accuracy: {acc}")
print("-"*30)
print(f"AUROC: {auroc}")
print("-"*30)
print(f"AUPRC: {auprc}")
print("-"*80)

true_labels = labels
predicted_labels = predictions

confusion_matrix = tf.math.confusion_matrix(
    true_labels,
    predicted_labels,
    num_classes=2,
    weights=None,
    name=None
)

sns.heatmap(confusion_matrix, annot=True, cmap='Blues', fmt='d')
sns.set (rc = {'figure.figsize':(5,5)})
plt.xlabel('Predicted Labels', fontsize=12)
plt.ylabel('True Labels', fontsize=12)
plt.yticks(np.arange(2)+0.5,('Soccer','Random'), fontsize="11")
plt.xticks(np.arange(2)+0.5,('Soccer','Random'), fontsize="11" )
plt.show()
55/55 [==============================] - 0s 1ms/step
events_vs_soccer metrics on real dataset:
Accuracy: 0.7181818181818181
------------------------------
AUROC: 0.7999999999999999
------------------------------
AUPRC: 0.2167832167832168
--------------------------------------------------------------------------------
No description has been provided for this image

Approach B - Unsupervised Learning¶

Evaluate the clustering algorithms of KMeans and Agglomerative Clustering for the binary task of distinguishing soccer related images and random images.

In [138]:
n_labels = np.argmax(labels_generic_events, axis=-1)
In [139]:
from sklearn.cluster import KMeans, AgglomerativeClustering

def clustering(
    data: np.array,
    n_clusters: int
):
    "Return clustering predictions with Kmeans and Agglomerative Clustering."
    
    clustering_predictions = {}
    clustering_predictions["KMeans"] = []
    clustering_predictions["Agglomerative"] = []
    
    K_clust = KMeans(n_clusters = n_clusters, random_state = 42, n_init="auto").fit(data)
    A_clust = AgglomerativeClustering(n_clusters = n_clusters).fit(data)

    clustering_predictions["KMeans"].append(K_clust.fit_predict(data))
    clustering_predictions["Agglomerative"].append(A_clust.fit_predict(data))
    
    return clustering_predictions
In [140]:
from sklearn.decomposition import PCA

def get_pca_decomposition(
    X: np.array
) -> pd.DataFrame:
    """Return the 2D PCA decomposition of the given data."""
    
    return PCA(n_components=2).fit_transform(X)
In [141]:
# RGB images
folderGenericEvents = r'D:\MachineLearning\Datasets\Soccer\GenericEvents' # Generic soccer and random events

data_generic_events_rgb, labels_generic_events_rgb = loadData(folderGenericEvents, True)

unsupervised_input_data_rgb_all = []
for event in data_generic_events_rgb:
    for element in data_generic_events_rgb[event]:
        unsupervised_input_data_rgb_all.append(element)
        
unsupervised_input_data_rgb_all = np.array(unsupervised_input_data_rgb_all)

# Gray scale images
unsupervised_input_data_all = []
for event in data_generic_events:
    for element in data_generic_events[event]:
        unsupervised_input_data_all.append(element)
        
unsupervised_input_data_all = np.array(unsupervised_input_data_all)
In [142]:
print("-- Input data for clustering --")
# Raw pixel values
raw_input = unsupervised_input_data_all.reshape(2600, -1)
print(f"Raw pixel values: {raw_input.shape}")

# Raw pixel values (PCA -> 2)
PCA_raw_input = get_pca_decomposition(raw_input)
print(f"Raw pixel values (PCA -> 2): {PCA_raw_input.shape}")

# Raw pixel values RGB
raw_input_rgb = unsupervised_input_data_rgb_all.reshape(2600, -1)
print(f"Raw pixel values RGB: {raw_input_rgb.shape}")

# Raw pixel values (PCA -> 2)
PCA_raw_input_rgb = get_pca_decomposition(raw_input_rgb)
print(f"Raw pixel values RGB (PCA -> 2): {PCA_raw_input_rgb.shape}")

# Representation extracted from pre trained model
threeChannels_input = np.array([cv2.cvtColor(x, cv2.COLOR_GRAY2RGB) for x in unsupervised_input_data_all])
image_shape = threeChannels_input.shape[1:]

resNet_feature_extraction = tf.keras.applications.resnet.ResNet50(input_shape=image_shape,
                                               include_top=False,
                                               weights='imagenet')
resNet_feature_extraction.trainable = False

# Gray images
features = get_predictions(resNet_feature_extraction, threeChannels_input, 64).reshape(2600, -1)
print(f"CNN features: {features.shape}")

# RGB images
features_rgb = get_predictions(resNet_feature_extraction, unsupervised_input_data_rgb_all, 64).reshape(2600, -1)
print(f"CNN features RGB: {features_rgb.shape}")

# Representation extracted from pre trained model + avg pooling
inputs = tf.keras.Input(shape=image_shape)
x = resNet_feature_extraction(inputs)
outputs = layers.GlobalAveragePooling2D()(x)
resNet_feature_extraction_pool = tf.keras.Model(inputs, outputs)

# Gray images
features_pooling = get_predictions(resNet_feature_extraction_pool, threeChannels_input, 64).reshape(2600, -1)
print(f"CNN features + pooling: {features_pooling.shape}")

# RGB images
features_pooling_rgb = get_predictions(resNet_feature_extraction_pool, unsupervised_input_data_rgb_all, 64).reshape(2600, -1)
print(f"CNN features + pooling: {features_pooling_rgb.shape}")

# (PCA -> 2) features
PCA_features = get_pca_decomposition(features)
print(f"CNN features (PCA -> 2): {PCA_features.shape}")

PCA_features_pooling = get_pca_decomposition(features_pooling)
print(f"CNN features + pooling (PCA -> 2): {PCA_features_pooling.shape}")

PCA_features_rgb = get_pca_decomposition(features_rgb)
print(f"CNN features RGB (PCA -> 2): {PCA_features_rgb.shape}")

PCA_features_pooling_rgb = get_pca_decomposition(features_pooling_rgb)
print(f"CNN features + pooling RGB (PCA -> 2): {PCA_features_pooling_rgb.shape}")

import skimage.feature

matrix_features = []
hog_features = []
for image in unsupervised_input_data_all:
    # Texture Features to capture texture patterns.
    # Calculate the co-occurrence matrix for the image and extract texture features
    co_matrix = skimage.feature.graycomatrix(image, [5], [0], levels=256)

    image_matrix_features = []
    contrast = skimage.feature.graycoprops(co_matrix, 'contrast')
    correlation = skimage.feature.graycoprops(co_matrix, 'correlation')
    energy = skimage.feature.graycoprops(co_matrix, 'energy')
    homogeneity = skimage.feature.graycoprops(co_matrix, 'homogeneity')
    
    image_matrix_features.append(contrast)
    image_matrix_features.append(correlation)
    image_matrix_features.append(energy)
    image_matrix_features.append(homogeneity)
    matrix_features.append(image_matrix_features)
    
    # Hog Features to capture object shapes and structures.
    from skimage.feature import hog
    from skimage import data, exposure

    hog_image = hog(
        image,
        orientations=5,
        pixels_per_cell=(10, 10),
        cells_per_block=(1, 1)
    )
    hog_features.append(hog_image)
    
matrix_features = np.array(matrix_features)
matrix_features = matrix_features.squeeze()
print(f"Co-occurrence matrix features {matrix_features.shape}")

PCA_matrix_features = get_pca_decomposition(matrix_features)
print(f"Co-occurrence matrix features (PCA -> 2) {PCA_matrix_features.shape}")

hog_features = np.array(hog_features)
print(f"HOG features {hog_features.shape}")

PCA_hog_features = get_pca_decomposition(hog_features)
print(f"HOG features (PCA -> 2) {PCA_hog_features.shape}")
-- Input data for clustering --
Raw pixel values: (2600, 6400)
Raw pixel values (PCA -> 2): (2600, 2)
Raw pixel values RGB: (2600, 19200)
Raw pixel values RGB (PCA -> 2): (2600, 2)
41/41 [==============================] - 1s 21ms/step
CNN features: (2600, 18432)
41/41 [==============================] - 1s 13ms/step
CNN features RGB: (2600, 18432)
41/41 [==============================] - 1s 14ms/step
CNN features + pooling: (2600, 2048)
41/41 [==============================] - 1s 14ms/step
CNN features + pooling: (2600, 2048)
CNN features (PCA -> 2): (2600, 2)
CNN features + pooling (PCA -> 2): (2600, 2)
CNN features RGB (PCA -> 2): (2600, 2)
CNN features + pooling RGB (PCA -> 2): (2600, 2)
Co-occurrence matrix features (2600, 4)
Co-occurrence matrix features (PCA -> 2) (2600, 2)
HOG features (2600, 320)
HOG features (PCA -> 2) (2600, 2)
In [143]:
predictions = {}
for input_data_name, input_data in (
    ("Raw", raw_input),
    ("PCA_raw", PCA_raw_input),
    ("Raw_rgb", raw_input_rgb),
    ("PCA_raw_rgb", PCA_raw_input_rgb),
    ("Features", features),
    ("Features+Pooling", features_pooling),
    ("Features_rgb", features_rgb),
    ("Features+Pooling_rgb", features_pooling_rgb),
    ("PCA_features", PCA_features),
    ("PCA_features_pooling", PCA_features_pooling),
    ("PCA_features_rgb", PCA_features_rgb),
    ("PCA_features_pooling_rgb", PCA_features_pooling_rgb),
    ("Matrix_features", matrix_features),
    ("PCA_matrix_features", PCA_matrix_features),
    ("Hog_features", hog_features),
    ("PCA_hog_features", PCA_hog_features)):
    
    predictions[input_data_name] = []
    predictions[input_data_name].append(clustering(input_data, 2))

Labels mismatch¶

This is binary clustering and the clustering algorithms are given the n_clusters=2 and always try to group datapoints (e.i don't output the same cluster for all datapoints).
Hence the label mismatch fix can be applied for an extrinsic evaluation by mapping clusters containing a majority of preditions of a label to that label.

In [144]:
import numpy as np

def fix_clustering_label_mismatch(true_labels, predicted_labels):
    """ Returns the predictions with the proper labels, a cluster represents the label most present in it."""

    swap = False
    
    # Change shape from (n_samples,) to (n_samples, n_classes)
    labels = label_binarize(true_labels, classes=np.unique(true_labels))
    predictions = label_binarize(predicted_labels, classes=np.unique(predicted_labels)) 
        
    # Check if there is a label mismatch (below 50% accuracy)
    if accuracy_score(labels, predictions) < 0.5:
        # Swapping label encoding to match majority in clusters
        fixed_predicted_labels = 1 - predicted_labels
        swap = True
        return true_labels, fixed_predicted_labels, swap
    
    return true_labels, predicted_labels, swap
In [145]:
from sklearn.metrics import mutual_info_score
from sklearn.metrics import adjusted_rand_score
from sklearn.metrics import mean_squared_error
from sklearn.metrics import silhouette_score

n_labels = np.argmax(labels_generic_events, axis=-1)

swap_list = {}
swap_list["KMeans"] = []
swap_list["Agglomerative"] = []

clustering_ex_results_KMeans = {"features":[], "accuracy":[], "AUROC":[], "AUPRC":[]}
clustering_ex_results_Agglomerative = {"features":[], "accuracy":[], "AUROC":[], "AUPRC":[]}

clustering_in_results_KMeans = {"features":[], "mif":[], "ars":[], "silhouette":[], "MSE":[]}
clustering_in_results_Agglomerative = {"features":[], "mif":[], "ars":[], "silhouette":[], "MSE":[]}

for data_type in predictions:
    print(f"-- Results for {data_type} input data --")
    for algorithm in predictions[data_type][0]:
        print(f"\n- Algorithm {algorithm}")
        
        true_labels = n_labels
        predicted_labels = predictions[data_type][0][algorithm][0]
        
        # Assign label to cluster where it's more present
        true_labels, predicted_labels, swap = fix_clustering_label_mismatch(true_labels, predicted_labels)
        
        # To visualize the correct labels in plot
        if "PCA" in data_type:
            swap_list[algorithm].append(swap)
        
        acc, auroc, auprc = get_metrics(
            labels = true_labels,
            predictions = predicted_labels
        )

        print("Extrinsic evaluation:")
        print("-"*40)
        print(f"Accuracy: {acc}")
        print("-"*30)
        print(f"AUROC: {auroc}")
        print("-"*30)
        print(f"AUPRC: {auprc}")
        print("-"*30)

        print("Intrinsic evaluation:")
        print("-"*40)
        mif = mutual_info_score(true_labels, predicted_labels)
        print(f"Mutual info score:   {mif}")
        ars = adjusted_rand_score(true_labels, predicted_labels)
        print(f"Adjusted rand score: {ars}")
        silhouette = silhouette_score(true_labels.reshape(-1, 1), predicted_labels)
        print(f"Silhouette score:    {silhouette}")
        MSE = mean_squared_error(true_labels, predicted_labels)
        print(f"Mean squared error:  {MSE}")
        
        if algorithm == "KMeans":
            clustering_ex_results_KMeans["features"].append(data_type)
            clustering_ex_results_KMeans["accuracy"].append(acc)
            clustering_ex_results_KMeans["AUROC"].append(auroc)
            clustering_ex_results_KMeans["AUPRC"].append(auprc)
            
            clustering_in_results_KMeans["features"].append(data_type)
            clustering_in_results_KMeans["mif"].append(mif)
            clustering_in_results_KMeans["ars"].append(ars)
            clustering_in_results_KMeans["silhouette"].append(silhouette)
            clustering_in_results_KMeans["MSE"].append(MSE)
        elif algorithm == "Agglomerative":
            clustering_ex_results_Agglomerative["features"].append({data_type})
            clustering_ex_results_Agglomerative["accuracy"].append(acc)
            clustering_ex_results_Agglomerative["AUROC"].append(auroc)
            clustering_ex_results_Agglomerative["AUPRC"].append(auprc)
            
            clustering_in_results_Agglomerative["features"].append({data_type})
            clustering_in_results_Agglomerative["mif"].append(mif)
            clustering_in_results_Agglomerative["ars"].append(ars)
            clustering_in_results_Agglomerative["silhouette"].append(silhouette)
            clustering_in_results_Agglomerative["MSE"].append(MSE)
                    
    print("-"*60)
    print("")
-- Results for Raw input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.595
------------------------------
AUROC: 0.596845238095238
------------------------------
AUPRC: 0.5193937701662944
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.01880166161812219
Adjusted rand score: 0.035725338741475494
Silhouette score:    0.06044244678260578
Mean squared error:  0.405

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6
------------------------------
AUROC: 0.5920238095238095
------------------------------
AUPRC: 0.5189239383804601
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.01775557547950879
Adjusted rand score: 0.03937162945708509
Silhouette score:    0.06613334601984541
Mean squared error:  0.4
------------------------------------------------------------

-- Results for PCA_raw input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5953846153846154
------------------------------
AUROC: 0.5970833333333333
------------------------------
AUPRC: 0.5195964604652945
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.018888676038459007
Adjusted rand score: 0.036019631076404844
Silhouette score:    0.0608944906765175
Mean squared error:  0.4046153846153846

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6307692307692307
------------------------------
AUROC: 0.6339880952380952
------------------------------
AUPRC: 0.5462151192994174
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.03636373290437567
Adjusted rand score: 0.06802537617769067
Silhouette score:    0.10856441587049105
Mean squared error:  0.36923076923076925
------------------------------------------------------------

-- Results for Raw_rgb input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6646153846153846
------------------------------
AUROC: 0.6766071428571427
------------------------------
AUPRC: 0.5753121833256565
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.0707387970876742
Adjusted rand score: 0.10765151135637231
Silhouette score:    0.16352301788164328
Mean squared error:  0.3353846153846154

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.571923076923077
------------------------------
AUROC: 0.5653571428571429
------------------------------
AUPRC: 0.49960563380281686
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.008787082874513796
Adjusted rand score: 0.020139715578809718
Silhouette score:    0.03554749848556304
Mean squared error:  0.4280769230769231
------------------------------------------------------------

-- Results for PCA_raw_rgb input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6592307692307692
------------------------------
AUROC: 0.6707142857142856
------------------------------
AUPRC: 0.5709124732583015
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.06529441580446627
Adjusted rand score: 0.10070470846049773
Silhouette score:    0.15420609019120057
Mean squared error:  0.34076923076923077

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6788461538461539
------------------------------
AUROC: 0.6870833333333334
------------------------------
AUPRC: 0.5861361669911314
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.07491400089767045
Adjusted rand score: 0.1274440748179099
Silhouette score:    0.1897416797960277
Mean squared error:  0.3211538461538462
------------------------------------------------------------

-- Results for Features input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6384615384615384
------------------------------
AUROC: 0.6203571428571428
------------------------------
AUPRC: 0.5517226598702503
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.0383717160877346
Adjusted rand score: 0.07510742071154326
Silhouette score:    0.11883614305999482
Mean squared error:  0.36153846153846153

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6480769230769231
------------------------------
AUROC: 0.6279761904761905
------------------------------
AUPRC: 0.5634561516914458
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.04718687865584606
Adjusted rand score: 0.085869496327039
Silhouette score:    0.13254317508013239
Mean squared error:  0.35192307692307695
------------------------------------------------------------

-- Results for Features+Pooling input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6280769230769231
------------------------------
AUROC: 0.6116666666666667
------------------------------
AUPRC: 0.5410440827038338
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.031122806870876513
Adjusted rand score: 0.06423658745430076
Silhouette score:    0.10376452389574349
Mean squared error:  0.3719230769230769

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6476923076923077
------------------------------
AUROC: 0.6278571428571429
------------------------------
AUPRC: 0.5627559857361182
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.046556339766477106
Adjusted rand score: 0.08545139153816873
Silhouette score:    0.13216906806865963
Mean squared error:  0.3523076923076923
------------------------------------------------------------

-- Results for Features_rgb input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.865
------------------------------
AUROC: 0.8588095238095238
------------------------------
AUPRC: 0.8157162628016407
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.30217324637488874
Adjusted rand score: 0.5326003592765147
Silhouette score:    0.617289254048675
Mean squared error:  0.135

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.8496153846153847
------------------------------
AUROC: 0.8561904761904762
------------------------------
AUPRC: 0.760267896587676
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.29384580410601846
Adjusted rand score: 0.4886899186838671
Silhouette score:    0.580269317468822
Mean squared error:  0.1503846153846154
------------------------------------------------------------

-- Results for Features+Pooling_rgb input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.8938461538461538
------------------------------
AUROC: 0.8915476190476191
------------------------------
AUPRC: 0.8426602564102564
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.3522049451977292
Adjusted rand score: 0.6202836720432763
Silhouette score:    0.6941226729781622
Mean squared error:  0.10615384615384615

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.8688461538461538
------------------------------
AUROC: 0.8711309523809524
------------------------------
AUPRC: 0.7931221343251275
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.30848743229302944
Adjusted rand score: 0.5440155194055404
Silhouette score:    0.6275085903607341
Mean squared error:  0.13115384615384615
------------------------------------------------------------

-- Results for PCA_features input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6411538461538462
------------------------------
AUROC: 0.6235714285714284
------------------------------
AUPRC: 0.554164689935615
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.03979640282204272
Adjusted rand score: 0.07818833106788244
Silhouette score:    0.12317512109095588
Mean squared error:  0.35884615384615387

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6488461538461539
------------------------------
AUROC: 0.6299404761904762
------------------------------
AUPRC: 0.5631297193620028
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.04636562937912242
Adjusted rand score: 0.08694913907617303
Silhouette score:    0.1346832120122245
Mean squared error:  0.35115384615384615
------------------------------------------------------------

-- Results for PCA_features_pooling input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.6303846153846154
------------------------------
AUROC: 0.6151785714285715
------------------------------
AUPRC: 0.5429844238330477
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.03205224661615563
Adjusted rand score: 0.06676229538776672
Silhouette score:    0.10729766349124768
Mean squared error:  0.3696153846153846

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5834615384615385
------------------------------
AUROC: 0.5777380952380953
------------------------------
AUPRC: 0.5078864368140262
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.01235832802781961
Adjusted rand score: 0.027352148304607197
Silhouette score:    0.047204080735378684
Mean squared error:  0.4165384615384615
------------------------------------------------------------

-- Results for PCA_features_rgb input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.8576923076923076
------------------------------
AUROC: 0.8517857142857143
------------------------------
AUPRC: 0.8036034353995519
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.2867895241234161
Adjusted rand score: 0.5114716590539701
Silhouette score:    0.5975968325478581
Mean squared error:  0.1423076923076923

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.8811538461538462
------------------------------
AUROC: 0.8780952380952382
------------------------------
AUPRC: 0.8269466364738444
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.3265340319183105
Adjusted rand score: 0.5809087445091459
Silhouette score:    0.6595567119005019
Mean squared error:  0.11884615384615385
------------------------------------------------------------

-- Results for PCA_features_pooling_rgb input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.8688461538461538
------------------------------
AUROC: 0.8651785714285712
------------------------------
AUPRC: 0.8113091917149033
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.3029963063488798
Adjusted rand score: 0.5439575487216932
Silhouette score:    0.6264752053285271
Mean squared error:  0.13115384615384615

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.7834615384615384
------------------------------
AUROC: 0.7687499999999999
------------------------------
AUPRC: 0.7293224299065422
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.19564189719120412
Adjusted rand score: 0.3204662160538807
Silhouette score:    0.41276437527141246
Mean squared error:  0.21653846153846154
------------------------------------------------------------

-- Results for Matrix_features input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5392307692307692
------------------------------
AUROC: 0.5241666666666667
------------------------------
AUPRC: 0.4745843935538592
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.0013742157295733026
Adjusted rand score: 0.004914628538723195
Silhouette score:    0.009990589102050728
Mean squared error:  0.46076923076923076

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5388461538461539
------------------------------
AUROC: 0.5089285714285715
------------------------------
AUPRC: 0.4663629053872957
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.00040243027451011826
Adjusted rand score: 0.002301566322516027
Silhouette score:    0.008719329356417064
Mean squared error:  0.46115384615384614
------------------------------------------------------------

-- Results for PCA_matrix_features input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5392307692307692
------------------------------
AUROC: 0.5241666666666667
------------------------------
AUPRC: 0.4745843935538592
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.0013742157295733026
Adjusted rand score: 0.004914628538723195
Silhouette score:    0.009990589102050728
Mean squared error:  0.46076923076923076

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5388461538461539
------------------------------
AUROC: 0.5089285714285715
------------------------------
AUPRC: 0.4663629053872957
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.00040243027451011826
Adjusted rand score: 0.002301566322516027
Silhouette score:    0.008719329356417064
Mean squared error:  0.46115384615384614
------------------------------------------------------------

-- Results for Hog_features input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5611538461538461
------------------------------
AUROC: 0.5808928571428571
------------------------------
AUPRC: 0.5064133777549974
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.017823099885256904
Adjusted rand score: 0.013212146934626497
Silhouette score:    0.016259148150338222
Mean squared error:  0.43884615384615383

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5634615384615385
------------------------------
AUROC: 0.5939880952380953
------------------------------
AUPRC: 0.5135696509140089
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.05595494473788535
Adjusted rand score: 0.01241278685242793
Silhouette score:    -0.07785522816668002
Mean squared error:  0.43653846153846154
------------------------------------------------------------

-- Results for PCA_hog_features input data --

- Algorithm KMeans
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5665384615384615
------------------------------
AUROC: 0.5838095238095238
------------------------------
AUPRC: 0.5084311867386319
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.017619918617434116
Adjusted rand score: 0.016297511554227886
Silhouette score:    0.02483166413958898
Mean squared error:  0.43346153846153845

- Algorithm Agglomerative
Extrinsic evaluation:
----------------------------------------
Accuracy: 0.5642307692307692
------------------------------
AUROC: 0.5720833333333333
------------------------------
AUPRC: 0.5020292532526576
------------------------------
Intrinsic evaluation:
----------------------------------------
Mutual info score:   0.010809198635083173
Adjusted rand score: 0.01592293040512831
Silhouette score:    0.02836691063325809
Mean squared error:  0.43576923076923074
------------------------------------------------------------

In [146]:
# 2D Plots
labels = np.unique(n_labels)

for algorithm in ("KMeans", "Agglomerative"):
    for data_type, decomposed_data, clustering_predictions, swap in (
        ("PCA_raw_input", PCA_raw_input, predictions["PCA_raw"][0][algorithm][0], swap_list[algorithm][0]),
        ("PCA_raw_input_rgb", PCA_raw_input_rgb, predictions["PCA_raw_rgb"][0][algorithm][0], swap_list[algorithm][1]),
        ("PCA_features", PCA_features, predictions["PCA_features"][0][algorithm][0], swap_list[algorithm][2]),
        ("PCA_features_pooling", PCA_features_pooling, predictions["PCA_features_pooling"][0][algorithm][0], swap_list[algorithm][3]),
        ("PCA_features_rgb", PCA_features_rgb, predictions["PCA_features_rgb"][0][algorithm][0], swap_list[algorithm][4]),
        ("PCA_features_pooling_rgb", PCA_features_pooling_rgb, predictions["PCA_features_pooling_rgb"][0][algorithm][0], swap_list[algorithm][5]),
        ("PCA_matrix_features", PCA_matrix_features, predictions["PCA_matrix_features"][0][algorithm][0], swap_list[algorithm][6]),
        ("PCA_hog_features", PCA_hog_features, predictions["PCA_hog_features"][0][algorithm][0], swap_list[algorithm][7])):


        plt.figure(figsize=(12,6))
        for l in labels:
            plt.scatter(decomposed_data[np.squeeze(n_labels == l), 0], decomposed_data[np.squeeze(n_labels == l), 1], label = l, marker = ".")
        plt.show()

        print(f"- {algorithm} predictions on {data_type} -")
        if swap:
            plt.figure(figsize=(12,6))
            for l in labels:
                plt.scatter(decomposed_data[clustering_predictions == 1-l, 0], decomposed_data[clustering_predictions == 1-l, 1], label = l, marker =".")
            plt.show() 
        else:
            plt.figure(figsize=(12,6))
            for l in labels:
                plt.scatter(decomposed_data[clustering_predictions == l, 0], decomposed_data[clustering_predictions == l, 1], label = l, marker =".")
            plt.show()

        print("-"*100)
No description has been provided for this image
- KMeans predictions on PCA_raw_input -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_raw_input_rgb -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_features -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_features_pooling -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_features_rgb -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_features_pooling_rgb -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_matrix_features -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- KMeans predictions on PCA_hog_features -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_raw_input -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_raw_input_rgb -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_features -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_features_pooling -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_features_rgb -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_features_pooling_rgb -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_matrix_features -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
No description has been provided for this image
- Agglomerative predictions on PCA_hog_features -
No description has been provided for this image
----------------------------------------------------------------------------------------------------
In [147]:
results_ex_clusteringDF_KMeans = pd.DataFrame(clustering_ex_results_KMeans)
results_ex_clusteringDF_Agglomerative = pd.DataFrame(clustering_ex_results_Agglomerative)

results_in_clusteringDF_KMeans = pd.DataFrame(clustering_in_results_KMeans)
results_in_clusteringDF_Agglomerative = pd.DataFrame(clustering_in_results_Agglomerative)
In [148]:
from barplots import barplots

barplots(
    results_ex_clusteringDF_KMeans,
    groupby=["features"],
    orientation="horizontal",
    height=12,
    legend_position="upper left"
)
                                                                                                                       
Out[148]:
[(<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'AUROC'}, xlabel='AUROC'>], dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Accuracy'}, xlabel='Accuracy'>],
        dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'AUPRC'}, xlabel='AUPRC'>], dtype=object))]
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [149]:
from barplots import barplots

barplots(
    results_ex_clusteringDF_Agglomerative,
    groupby=["features"],
    orientation="horizontal",
    height=12,
    legend_position="upper left"
)
                                                                                                                       
Out[149]:
[(<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'AUROC'}, xlabel='AUROC'>], dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Accuracy'}, xlabel='Accuracy'>],
        dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'AUPRC'}, xlabel='AUPRC'>], dtype=object))]
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [150]:
from barplots import barplots

barplots(
    results_in_clusteringDF_KMeans,
    groupby=["features"],
    orientation="horizontal",
    height=12,
    legend_position="upper left"
)
                                                                                                                       
Out[150]:
[(<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'MSE'}, xlabel='MSE'>], dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Silhouette'}, xlabel='Silhouette'>],
        dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Ars'}, xlabel='Ars'>], dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Mif'}, xlabel='Mif'>], dtype=object))]
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [151]:
from barplots import barplots

barplots(
    results_in_clusteringDF_Agglomerative,
    groupby=["features"],
    orientation="horizontal",
    height=12,
    legend_position="upper left"
)
                                                                                                                       
Out[151]:
[(<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'MSE'}, xlabel='MSE'>], dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Silhouette'}, xlabel='Silhouette'>],
        dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Ars'}, xlabel='Ars'>], dtype=object)),
 (<Figure size 2400x960 with 1 Axes>,
  array([<Axes: title={'center': 'Mif'}, xlabel='Mif'>], dtype=object))]
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image